iicd/MeCo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
4df61fcbb3
MeCo/notebooks_201/op_strength.ipynb

1613 lines
1.3 MiB
Plaintext
Raw Normal View History

upload
2023-05-04 07:09:03 +02:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 131,
"id": "78dba395",
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"import pickle\n",
"import statistics as stat"
]
},
{
"cell_type": "code",
"execution_count": 132,
"id": "a2566b68",
"metadata": {},
"outputs": [],
"source": [
"m_names = {\n",
" 'acc':'avg-acc',\n",
" 'best-acc':'best-acc',\n",
" 'discrete acc darts':'disc-acc',\n",
" 'best_nwot':'best_zc(nwot)',\n",
" 'best_synflow':'best_zc(synflow)',\n",
" 'dartspt':'darts-pt',\n",
" 'zc pt':'zc-pt(nwot)',\n",
" 'discrete zc':'disc-zc(nwot)',\n",
" #'alpha':'darts-100',\n",
" 'alpha-60':'darts',\n",
" 'tenaspt':'tenas',\n",
" 'synflow':'zc-pt(synflow)',\n",
" 'synflow_disc':'disc-zc(synflow)',\n",
" 'zc-pt-post':'zc-pt-post',\n",
" 'zc-disc-post':'zc-disc-post'\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 133,
"id": "7cc2961d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3']\n"
]
}
],
"source": [
"ops=[]\n",
"edge = 0\n",
"num_edges = 6\n",
"op_strengths = []\n",
"for e in range(num_edges):\n",
" op_strengths.append({})\n",
"\n",
"with open('op_strength_src.csv') as csv_file:\n",
" csv_reader = csv.reader(csv_file, delimiter=',')\n",
" line_count = 0\n",
" for row in csv_reader:\n",
" if line_count == 0:\n",
" ops = row[1:-1]\n",
" line_count += 1\n",
" else:\n",
" #print(row)\n",
" if row == ['', '', '', '', '', '', '']:\n",
" edge += 1\n",
" line_count += 1\n",
" continue\n",
" #parse row\n",
" if row[0] not in m_names:\n",
" continue\n",
" method = m_names[row[0]]\n",
" op_strengths[edge][method] = []\n",
" for r in row[1:-1]:\n",
" if '±' in r:\n",
" r1 = r.split('±')[0]\n",
" r2 = r.split('±')[1]\n",
" r=r1\n",
" metric = float(r)\n",
" op_strengths[edge][method].append(metric)\n",
" line_count += 1\n",
"print(ops)"
]
},
{
"cell_type": "code",
"execution_count": 134,
"id": "7ececab5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'avg-acc': [77.36, 83.81, 86.38, 87.32, 81.02],\n",
" 'disc-acc': [83.42, 84.1, 72.0, 76.35, 39.66],\n",
" 'darts-pt': [85.43, 17.02, 78.13, 59.09, 85.34],\n",
" 'zc-pt(nwot)': [3455.233646,\n",
" 3449.898772,\n",
" 3449.538363,\n",
" 3441.815563,\n",
" 3461.179476],\n",
" 'disc-zc(nwot)': [3331.007285,\n",
" 3445.489455,\n",
" 3366.877065,\n",
" 3437.551079,\n",
" 3423.180255],\n",
" 'best-acc': [94.15, 94.18, 94.44, 94.68, 93.86],\n",
" 'darts': [0.1387, 0.4758, 0.1296, 0.181, 0.0748],\n",
" 'tenas': [38.5, 48.0, 31.0, 6.0, 37.5],\n",
" 'zc-pt(synflow)': [1.9286723850908796e+31,\n",
" 7.990282869734622e+30,\n",
" 1.2421187150331997e+30,\n",
" 9.438907569335487e+26,\n",
" 8.191504786187086e+30],\n",
" 'disc-zc(synflow)': [4.639162000716631e+21,\n",
" 1.4975281050055959e+26,\n",
" 4.2221622054263117e+30,\n",
" 1.9475517523688712e+36,\n",
" 1.5075022033622535e+26],\n",
" 'best_zc(nwot)': [1702.1967536035393,\n",
" 1773.1779654806287,\n",
" 1793.8140278364453,\n",
" 1792.8682630835763,\n",
" 1761.1262357119376],\n",
" 'best_zc(synflow)': [5.784248799475683e+39,\n",
" 1.4769546208886953e+44,\n",
" 6.658953754065702e+49,\n",
" 5.1987025703231504e+39,\n",
" 1.9928388494681343e+35],\n",
" 'zc-pt-post': [3067.0476, 3055.9404, 3059.8901, 3060.4536, 3073.5583],\n",
" 'zc-disc-post': [2942.267, 3068.6416, 3009.5847, 3028.1794, 3031.4248]},\n",
" {'avg-acc': [80.03, 83.11, 85.23, 85.99, 81.52],\n",
" 'disc-acc': [85.12, 83.39, 76.72, 81.34, 84.38],\n",
" 'darts-pt': [85.52, 36.1, 84.39, 80.95, 85.49],\n",
" 'zc-pt(nwot)': [3452.145851,\n",
" 3448.696318,\n",
" 3441.809174,\n",
" 3440.652631,\n",
" 3453.739943],\n",
" 'disc-zc(nwot)': [3429.074707,\n",
" 3435.750274,\n",
" 3407.872847,\n",
" 3434.584512,\n",
" 3421.442414],\n",
" 'best-acc': [94.24, 94.16, 94.49, 94.68, 94.09],\n",
" 'darts': [0.1236, 0.5535, 0.11, 0.1249, 0.088],\n",
" 'tenas': [7.0, 55.0, 10.0, 15.0, 39.0],\n",
" 'zc-pt(synflow)': [3.116079880492518e+30,\n",
" 2.5018418732419554e+30,\n",
" 1.4274537256246266e+30,\n",
" 3.138287824323275e+29,\n",
" 2.5693894962958226e+30],\n",
" 'disc-zc(synflow)': [5.615386425664938e+28,\n",
" 2.340336657109326e+29,\n",
" 1.9258305801684058e+30,\n",
" 3.012759514473006e+32,\n",
" 2.2897138361934977e+29],\n",
" 'best_zc(nwot)': [1765.3743820515451,\n",
" 1770.8436009213751,\n",
" 1791.917305624048,\n",
" 1793.8140278364453,\n",
" 1763.877253730585],\n",
" 'best_zc(synflow)': [1.9424580089849912e+49,\n",
" 2.764587447411338e+49,\n",
" 6.658953754065702e+49,\n",
" 2.0353792445711388e+49,\n",
" 1.4435653786128956e+49],\n",
" 'zc-pt-post': [3067.0476, 3058.9197, 3048.8745, 3051.2664, 3066.668],\n",
" 'zc-disc-post': [3020.0203, 3052.1936, 3026.2217, 3022.0935, 3029.2]},\n",
" {'avg-acc': [82.9, 82.44, 84.05, 84.49, 81.98],\n",
" 'disc-acc': [85.96, 85.18, 54.02, 78.41, 84.88],\n",
" 'darts-pt': [85.51, 80.29, 81.86, 77.68, 85.32],\n",
" 'zc-pt(nwot)': [3446.521007,\n",
" 3447.612434,\n",
" 3435.455206,\n",
" 3436.396744,\n",
" 3449.275466],\n",
" 'disc-zc(nwot)': [3428.795464,\n",
" 3423.361285,\n",
" 3440.925616,\n",
" 3437.286935,\n",
" 3416.891544],\n",
" 'best-acc': [94.25, 94.43, 94.49, 94.68, 94.19],\n",
" 'darts': [0.2403, 0.3297, 0.1495, 0.1748, 0.1056],\n",
" 'tenas': [31.5, 10.0, 30.0, 16.5, 36.5],\n",
" 'zc-pt(synflow)': [1.0312338471669537e+31,\n",
" 4.9191575008661263e+30,\n",
" 1.4241158958667068e+30,\n",
" 1.0282498082879338e+28,\n",
" 5.038622256524752e+30],\n",
" 'disc-zc(synflow)': [1.6980829611704765e+25,\n",
" 3.3199508659283994e+27,\n",
" 3.3825056097270114e+30,\n",
" 1.2059727722928161e+35,\n",
" 3.279653417965715e+27],\n",
" 'best_zc(nwot)': [1764.51075805859,\n",
" 1764.116749555202,\n",
" 1793.8140278364453,\n",
" 1792.8239766388833,\n",
" 1764.1848313456592],\n",
" 'best_zc(synflow)': [8.376122028137071e+41,\n",
" 1.0615041036082487e+45,\n",
" 6.658953754065702e+49,\n",
" 8.399427750574918e+41,\n",
" 2.5270360875229e+39],\n",
" 'zc-pt-post': [3067.0476, 3068.708, 3056.3506, 3047.9695, 3071.3577],\n",
" 'zc-disc-post': [3044.023, 3033.0627, 3032.825, 3052.0688, 3023.2302]},\n",
" {'avg-acc': [74.02, 85.17, 87.3, 88.28, 81.38],\n",
" 'disc-acc': [66.18, 85.38, 78.8, 81.59, 82.89],\n",
" 'darts-pt': [85.49, 9.86, 81.79, 59.18, 85.48],\n",
" 'zc-pt(nwot)': [3453.805194,\n",
" 3435.985406,\n",
" 3444.044047,\n",
" 3445.595326,\n",
" 3447.067855],\n",
" 'disc-zc(nwot)': [3408.990502,\n",
" 3464.050741,\n",
" 3359.888463,\n",
" 3382.1755,\n",
" 3431.805571],\n",
" 'best-acc': [94.16, 94.68, 94.03, 94.04, 93.85],\n",
" 'darts': [0.0636, 0.6513, 0.0826, 0.1335, 0.0691],\n",
" 'tenas': [34.0, 44.0, 53.5, 23.0, 30.0],\n",
" 'zc-pt(synflow)': [2.0042808467776213e+30,\n",
" 1.9513599734483263e+30,\n",
" 1.5188352495143643e+30,\n",
" 7.704103863066581e+29,\n",
" 1.9536326167605112e+30],\n",
" 'disc-zc(synflow)': [4.3050000047616484e+29,\n",
" 7.635399455155384e+29,\n",
" 1.5949429556375966e+30,\n",
" 1.4519088590209463e+31,\n",
" 7.345232988374157e+29],\n",
" 'best_zc(nwot)': [1766.5481959337162,\n",
" 1769.1683503033412,\n",
" 1793.8140278364453,\n",
" 1792.8682630835763,\n",
" 1765.1445530390838],\n",
" 'best_zc(synflow)': [5.90523769961745e+49,\n",
" 6.344766865099622e+49,\n",
" 6.571181309028854e+49,\n",
" 6.57509920946309e+49,\n",
" 6.658953754065702e+49],\n",
" 'zc-pt-post': [3067.0476, 3032.6658, 3058.9646, 3059.2861, 3047.1965],\n",
" 'zc-disc-post': [2975.976, 3130.7397, 3008.5625, 3009.341, 3086.3398]},\n",
" {'avg-acc': [80.14, 83.05, 85.09, 85.7, 81.89],\n",
" 'disc-acc': [86.44, 84.75, 80.23, 80.46, 80.13],\n",
" 'darts-pt': [85.45, 51.15, 78.84, 64.64, 85.14],\n",
" 'zc-pt(nwot)': [3451.055723,\n",
" 3449.796894,\n",
" 3442.625354,\n",
" 3441.131751,\n",
" 3453.311493],\n",
" 'disc-zc(nwot)': [3433.98773,\n",
" 3435.573458,\n",
" 3424.470031,\n",
" 3431.143217,\n",
" 3423.153213],\n",
" 'best-acc': [94.29, 94.18, 94.56, 94.68, 94.23],\n",
" 'darts': [0.1183, 0.48, 0.1305, 0.1732, 0.0979],\n",
" 'tenas': [32.0, 32.5, 36.5, 32.0, 52.0],\n",
" 'zc-pt(synflow)': [3.165975343348193e+30,\n",
" 2.4302742111297496e+30,\n",
" 1.4853908452542004e+30,\n",
" 2.868307126123347e+29,\n",
" 2.6891361283692336e+30],\n",
" 'disc-zc(synflow)': [5.5202846896598e+28,\n",
" 2.4896852024898197e+29,\n",
" 2.1810394989246777e+30,\n",
" 2.9482018739806336e+32,\n",
" 2.2732178076450144e+29],\n",
" 'best_zc(nwot)': [1752.024924623228,\n",
" 1793.8140278364453,\n",
" 1786.3402409418215,\n",
" 1785.0294182838636,\n",
" 1781.9741301640186],\n",
" 'best_zc(synflow)': [1.8865959738805548e+49,\n",
" 2.593134717306188e+49,\n",
" 6.658953754065702e+49,\n",
" 2.021273089103704e+49,\n",
" 1.6187260144154453e+49],\n",
" 'zc-pt-post': [3067.0476, 3060.9983, 3057.1006, 3054.3428, 3066.2087],\n",
" 'zc-disc-post': [3037.8726, 3055.4219, 3027.6638, 3024.3271, 3037.8108]},\n",
" {'avg-acc': [77.61, 83.43, 86.18, 86.95, 81.74],\n",
" 'disc-acc': [86.28, 82.69, 77.13, 76.8, 81.99],\n",
" 'darts-pt': [85.54, 32.43, 81.04, 72.75, 85.51],\n",
" 'zc-pt(nwot)': [3450.967554,\n",
" 3448.211459,\n",
" 3440.79926,\n",
" 3443.240243,\n",
" 3452.989921],\n",
" 'disc-zc(nwot)': [3434.421701,\n",
" 3437.661196,\n",
" 3418.572637,\n",
" 3397.51709,\n",
" 3424.166157],\n",
" 'best-acc': [94.05, 94.16, 94.68, 94.56, 94.1],\n",
" 'darts': [0.1648, 0.4853, 0.1223, 0.1397, 0.088],\n",
" 'tenas': [38.5, 16.0, 20.0, 17.0, 27.5],\n",
" 'zc-pt(synflow)': [1.9460309216168614e+31,\n",
" 8.014786854561015e+30,\n",
" 1.1851807660289746e+30,\n",
" 8.96867143875011e+26,\n",
" 7.75842932776677e+30],\n",
" 'disc-zc(synflow)': [4.777733726551756e+21,\n",
" 1.4572459237815469e+26,\n",
" 3.8590321292364994e+30,\n",
" 1.8898449210848245e+36,\n",
" 1.5222938895812008e+26],\n",
" 'best_zc(nwot)': [1761.8789642379636,\n",
" 1769.103803678444,\n",
" 1793.8140278364453,\n",
" 1792.8239766388833,\n",
" 1761.9145207476113],\n",
" 'best_zc(synflow)': [5.776473195639679e+39,\n",
" 1.4672616553030765e+44,\n",
" 6.658953754065702e+49,\n",
" 5.480408193532999e+39,\n",
" 1.9606567871518125e+35],\n",
" 'zc-pt-post': [3067.0476, 3063.1135, 3058.818, 3064.5405, 3070.7593],\n",
" 'zc-disc-post': [3061.6133, 3063.294, 3038.05, 3012.938, 3042.5535]}]"
]
},
"execution_count": 134,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"op_strengths"
]
},
{
"cell_type": "code",
"execution_count": 135,
"id": "566fe377",
"metadata": {},
"outputs": [],
"source": [
"#override disc_acc with results from lukasz\n",
"disc_accuracy_stage_0 = {(100, 0, 0): 83.267998,\n",
" (100, 0, 1): 82.239998,\n",
" (100, 0, 2): 64.995995,\n",
" (100, 0, 3): 71.755997,\n",
" (100, 0, 4): 54.307999,\n",
" (100, 1, 0): 84.935997,\n",
" (100, 1, 1): 83.231995,\n",
" (100, 1, 2): 73.231995,\n",
" (100, 1, 3): 76.771996,\n",
" (100, 1, 4): 83.451996,\n",
" (100, 2, 0): 83.867996,\n",
" (100, 2, 1): 83.727997,\n",
" (100, 2, 2): 77.332001,\n",
" (100, 2, 3): 76.832001,\n",
" (100, 2, 4): 83.251999,\n",
" (100, 3, 0): 65.767998,\n",
" (100, 3, 1): 84.444,\n",
" (100, 3, 2): 75.816002,\n",
" (100, 3, 3): 78.68,\n",
" (100, 3, 4): 62.703999,\n",
" (100, 4, 0): 83.571999,\n",
" (100, 4, 1): 82.028,\n",
" (100, 4, 2): 75.023994,\n",
" (100, 4, 3): 76.087997,\n",
" (100, 4, 4): 82.563995,\n",
" (100, 5, 0): 83.947998,\n",
" (100, 5, 1): 82.447998,\n",
" (100, 5, 2): 66.692001,\n",
" (100, 5, 3): 71.363998,\n",
" (100, 5, 4): 80.311996}\n",
"\n",
"for e in range(6):\n",
" for o in range(5):\n",
" op_strengths[e]['disc-acc'][o] = disc_accuracy_stage_0[(100,e,o)]"
]
},
{
"cell_type": "code",
"execution_count": 136,
"id": "c9d1fcc3",
"metadata": {},
"outputs": [],
"source": [
"flat_op_strengths = {}\n",
"for k in op_strengths[0].keys():\n",
" flat_op_strengths[k] = []\n",
"for e in op_strengths:\n",
" for k,v in e.items():\n",
" if 'pt' in k or 'tenas' in k:\n",
" flat_op_strengths[k].extend([-vv for vv in v])\n",
" else:\n",
" flat_op_strengths[k].extend(v)"
]
},
{
"cell_type": "code",
"execution_count": 137,
"id": "b5a2757c",
"metadata": {},
"outputs": [],
"source": [
"sq_op_strengths = {}\n",
"for k in op_strengths[0].keys():\n",
" sq_op_strengths[k] = []\n",
"for e in op_strengths:\n",
" for k,v in e.items():\n",
" if 'pt' in k or 'tenas' in k:\n",
" sq_op_strengths[k].append([-vv for vv in v])\n",
" else:\n",
" sq_op_strengths[k].append(v)"
]
},
{
"cell_type": "code",
"execution_count": 138,
"id": "efeb1742",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'avg-acc': [77.36, 83.81, 86.38, 87.32, 81.02], 'disc-acc': [83.267998, 82.239998, 64.995995, 71.755997, 54.307999], 'darts-pt': [-85.43, -17.02, -78.13, -59.09, -85.34], 'zc-pt(nwot)': [-3455.233646, -3449.898772, -3449.538363, -3441.815563, -3461.179476], 'disc-zc(nwot)': [3331.007285, 3445.489455, 3366.877065, 3437.551079, 3423.180255], 'best-acc': [94.15, 94.18, 94.44, 94.68, 93.86], 'darts': [0.1387, 0.4758, 0.1296, 0.181, 0.0748], 'tenas': [-38.5, -48.0, -31.0, -6.0, -37.5], 'zc-pt(synflow)': [-1.9286723850908796e+31, -7.990282869734622e+30, -1.2421187150331997e+30, -9.438907569335487e+26, -8.191504786187086e+30], 'disc-zc(synflow)': [4.639162000716631e+21, 1.4975281050055959e+26, 4.2221622054263117e+30, 1.9475517523688712e+36, 1.5075022033622535e+26], 'best_zc(nwot)': [1702.1967536035393, 1773.1779654806287, 1793.8140278364453, 1792.8682630835763, 1761.1262357119376], 'best_zc(synflow)': [5.784248799475683e+39, 1.4769546208886953e+44, 6.658953754065702e+49, 5.1987025703231504e+39, 1.9928388494681343e+35], 'zc-pt-post': [-3067.0476, -3055.9404, -3059.8901, -3060.4536, -3073.5583], 'zc-disc-post': [2942.267, 3068.6416, 3009.5847, 3028.1794, 3031.4248]}\n",
"{'avg-acc': [80.03, 83.11, 85.23, 85.99, 81.52], 'disc-acc': [84.935997, 83.231995, 73.231995, 76.771996, 83.451996], 'darts-pt': [-85.52, -36.1, -84.39, -80.95, -85.49], 'zc-pt(nwot)': [-3452.145851, -3448.696318, -3441.809174, -3440.652631, -3453.739943], 'disc-zc(nwot)': [3429.074707, 3435.750274, 3407.872847, 3434.584512, 3421.442414], 'best-acc': [94.24, 94.16, 94.49, 94.68, 94.09], 'darts': [0.1236, 0.5535, 0.11, 0.1249, 0.088], 'tenas': [-7.0, -55.0, -10.0, -15.0, -39.0], 'zc-pt(synflow)': [-3.116079880492518e+30, -2.5018418732419554e+30, -1.4274537256246266e+30, -3.138287824323275e+29, -2.5693894962958226e+30], 'disc-zc(synflow)': [5.615386425664938e+28, 2.340336657109326e+29, 1.9258305801684058e+30, 3.012759514473006e+32, 2.2897138361934977e+29], 'best_zc(nwot)': [1765.3743820515451, 1770.8436009213751, 1791.917305624048, 1793.8140278364453, 1763.877253730585], 'best_zc(synflow)': [1.9424580089849912e+49, 2.764587447411338e+49, 6.658953754065702e+49, 2.0353792445711388e+49, 1.4435653786128956e+49], 'zc-pt-post': [-3067.0476, -3058.9197, -3048.8745, -3051.2664, -3066.668], 'zc-disc-post': [3020.0203, 3052.1936, 3026.2217, 3022.0935, 3029.2]}\n",
"{'avg-acc': [82.9, 82.44, 84.05, 84.49, 81.98], 'disc-acc': [83.867996, 83.727997, 77.332001, 76.832001, 83.251999], 'darts-pt': [-85.51, -80.29, -81.86, -77.68, -85.32], 'zc-pt(nwot)': [-3446.521007, -3447.612434, -3435.455206, -3436.396744, -3449.275466], 'disc-zc(nwot)': [3428.795464, 3423.361285, 3440.925616, 3437.286935, 3416.891544], 'best-acc': [94.25, 94.43, 94.49, 94.68, 94.19], 'darts': [0.2403, 0.3297, 0.1495, 0.1748, 0.1056], 'tenas': [-31.5, -10.0, -30.0, -16.5, -36.5], 'zc-pt(synflow)': [-1.0312338471669537e+31, -4.9191575008661263e+30, -1.4241158958667068e+30, -1.0282498082879338e+28, -5.038622256524752e+30], 'disc-zc(synflow)': [1.6980829611704765e+25, 3.3199508659283994e+27, 3.3825056097270114e+30, 1.2059727722928161e+35, 3.279653417965715e+27], 'best_zc(nwot)': [1764.51075805859, 1764.116749555202, 1793.8140278364453, 1792.8239766388833, 1764.1848313456592], 'best_zc(synflow)': [8.376122028137071e+41, 1.0615041036082487e+45, 6.658953754065702e+49, 8.399427750574918e+41, 2.5270360875229e+39], 'zc-pt-post': [-3067.0476, -3068.708, -3056.3506, -3047.9695, -3071.3577], 'zc-disc-post': [3044.023, 3033.0627, 3032.825, 3052.0688, 3023.2302]}\n",
"{'avg-acc': [74.02, 85.17, 87.3, 88.28, 81.38], 'disc-acc': [65.767998, 84.444, 75.816002, 78.68, 62.703999], 'darts-pt': [-85.49, -9.86, -81.79, -59.18, -85.48], 'zc-pt(nwot)': [-3453.805194, -3435.985406, -3444.044047, -3445.595326, -3447.067855], 'disc-zc(nwot)': [3408.990502, 3464.050741, 3359.888463, 3382.1755, 3431.805571], 'best-acc': [94.16, 94.68, 94.03, 94.04, 93.85], 'darts': [0.0636, 0.6513, 0.0826, 0.1335, 0.0691], 'tenas': [-34.0, -44.0, -53.5, -23.0, -30.0], 'zc-pt(synflow)': [-2.0042808467776213e+30, -1.9513599734483263e+30, -1.5188352495143643e+30, -7.704103863066581e+29, -1.9536326167605112e+30], 'disc-zc(synflow)': [4.3050000047616484e+29, 7.635399455155384e+29, 1.5949429556375966e+30, 1.4519088590209463e+31, 7.345232988374157e+29], 'best_zc(nwot)': [1766.5481959337162, 1769.1683503033412, 1793.8140278364453, 1792.8682630835763, 1765.1445530390838], 'best_zc(synflow)': [5.90523769961745e+49, 6.344766865099622e+49, 6.571181309028854e+49, 6.57509920946309e+49, 6.658953754065702e+49], 'zc-pt-post': [-3067.0476, -3032.6658, -3058.9646, -3059.2861, -3047.1965], 'zc-disc-post': [2975.976, 3130.7397, 3008.5625, 3009.341, 3086.3398]}\n",
"{'avg-acc': [80.14, 83.05, 85.09, 85.7, 81.89], 'disc-acc': [83.571999, 82.028, 75.023994, 76.087997, 82.563995], 'darts-pt': [-85.45, -51.15, -78.84, -64.64, -85.14], 'zc-pt(nwot)': [-3451.055723, -3449.796894, -3442.625354, -3441.131751, -3453.311493], 'disc-zc(nwot)': [3433.98773, 3435.573458, 3424.470031, 3431.143217, 3423.153213], 'best-acc': [94.29, 94.18, 94.56, 94.68, 94.23], 'darts': [0.1183, 0.48, 0.1305, 0.1732, 0.0979], 'tenas': [-32.0, -32.5, -36.5, -32.0, -52.0], 'zc-pt(synflow)': [-3.165975343348193e+30, -2.4302742111297496e+30, -1.4853908452542004e+30, -2.868307126123347e+29, -2.6891361283692336e+30], 'disc-zc(synflow)': [5.5202846896598e+28, 2.4896852024898197e+29, 2.1810394989246777e+30, 2.9482018739806336e+32, 2.2732178076450144e+29], 'best_zc(nwot)': [1752.024924623228, 1793.8140278364453, 1786.3402409418215, 1785.0294182838636, 1781.9741301640186], 'best_zc(synflow)': [1.8865959738805548e+49, 2.593134717306188e+49, 6.658953754065702e+49, 2.021273089103704e+49, 1.6187260144154453e+49], 'zc-pt-post': [-3067.0476, -3060.9983, -3057.1006, -3054.3428, -3066.2087], 'zc-disc-post': [3037.8726, 3055.4219, 3027.6638, 3024.3271, 3037.8108]}\n",
"{'avg-acc': [77.61, 83.43, 86.18, 86.95, 81.74], 'disc-acc': [83.947998, 82.447998, 66.692001, 71.363998, 80.311996], 'darts-pt': [-85.54, -32.43, -81.04, -72.75, -85.51], 'zc-pt(nwot)': [-3450.967554, -3448.211459, -3440.79926, -3443.240243, -3452.989921], 'disc-zc(nwot)': [3434.421701, 3437.661196, 3418.572637, 3397.51709, 3424.166157], 'best-acc': [94.05, 94.16, 94.68, 94.56, 94.1], 'darts': [0.1648, 0.4853, 0.1223, 0.1397, 0.088], 'tenas': [-38.5, -16.0, -20.0, -17.0, -27.5], 'zc-pt(synflow)': [-1.9460309216168614e+31, -8.014786854561015e+30, -1.1851807660289746e+30, -8.96867143875011e+26, -7.75842932776677e+30], 'disc-zc(synflow)': [4.777733726551756e+21, 1.4572459237815469e+26, 3.8590321292364994e+30, 1.8898449210848245e+36, 1.5222938895812008e+26], 'best_zc(nwot)': [1761.8789642379636, 1769.103803678444, 1793.8140278364453, 1792.8239766388833, 1761.9145207476113], 'best_zc(synflow)': [5.776473195639679e+39, 1.4672616553030765e+44, 6.658953754065702e+49, 5.480408193532999e+39, 1.9606567871518125e+35], 'zc-pt-post': [-3067.0476, -3063.1135, -3058.818, -3064.5405, -3070.7593], 'zc-disc-post': [3061.6133, 3063.294, 3038.05, 3012.938, 3042.5535]}\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>avg-acc</th>\n",
" <th>disc-acc</th>\n",
" <th>darts-pt</th>\n",
" <th>zc-pt(nwot)</th>\n",
" <th>disc-zc(nwot)</th>\n",
" <th>best-acc</th>\n",
" <th>darts</th>\n",
" <th>tenas</th>\n",
" <th>zc-pt(synflow)</th>\n",
" <th>disc-zc(synflow)</th>\n",
" <th>best_zc(nwot)</th>\n",
" <th>best_zc(synflow)</th>\n",
" <th>zc-pt-post</th>\n",
" <th>zc-disc-post</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>avg-acc</th>\n",
" <td>1.000000</td>\n",
" <td>-0.483333</td>\n",
" <td>0.666667</td>\n",
" <td>0.833333</td>\n",
" <td>0.033333</td>\n",
" <td>0.650000</td>\n",
" <td>0.316667</td>\n",
" <td>0.300433</td>\n",
" <td>0.933333</td>\n",
" <td>0.916667</td>\n",
" <td>0.816667</td>\n",
" <td>0.416667</td>\n",
" <td>0.683333</td>\n",
" <td>-0.033333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>best-acc</th>\n",
" <td>0.650000</td>\n",
" <td>-0.316667</td>\n",
" <td>0.416667</td>\n",
" <td>0.766667</td>\n",
" <td>0.100000</td>\n",
" <td>1.000000</td>\n",
" <td>0.200000</td>\n",
" <td>0.435065</td>\n",
" <td>0.633333</td>\n",
" <td>0.600000</td>\n",
" <td>0.533333</td>\n",
" <td>0.250000</td>\n",
" <td>0.583333</td>\n",
" <td>-0.250000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>best_zc(nwot)</th>\n",
" <td>0.816667</td>\n",
" <td>-0.466667</td>\n",
" <td>0.550000</td>\n",
" <td>0.783333</td>\n",
" <td>0.066667</td>\n",
" <td>0.533333</td>\n",
" <td>0.216667</td>\n",
" <td>0.099134</td>\n",
" <td>0.750000</td>\n",
" <td>0.733333</td>\n",
" <td>1.000000</td>\n",
" <td>0.466667</td>\n",
" <td>0.583333</td>\n",
" <td>-0.100000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>best_zc(synflow)</th>\n",
" <td>0.416667</td>\n",
" <td>-0.350000</td>\n",
" <td>0.400000</td>\n",
" <td>0.533333</td>\n",
" <td>0.166667</td>\n",
" <td>0.250000</td>\n",
" <td>0.366667</td>\n",
" <td>0.225216</td>\n",
" <td>0.383333</td>\n",
" <td>0.333333</td>\n",
" <td>0.466667</td>\n",
" <td>1.000000</td>\n",
" <td>0.650000</td>\n",
" <td>0.100000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>darts</th>\n",
" <td>0.316667</td>\n",
" <td>0.350000</td>\n",
" <td>0.650000</td>\n",
" <td>0.450000</td>\n",
" <td>0.500000</td>\n",
" <td>0.200000</td>\n",
" <td>1.000000</td>\n",
" <td>0.135065</td>\n",
" <td>0.216667</td>\n",
" <td>0.150000</td>\n",
" <td>0.216667</td>\n",
" <td>0.366667</td>\n",
" <td>0.400000</td>\n",
" <td>0.400000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>darts-pt</th>\n",
" <td>0.666667</td>\n",
" <td>-0.216667</td>\n",
" <td>1.000000</td>\n",
" <td>0.583333</td>\n",
" <td>0.400000</td>\n",
" <td>0.416667</td>\n",
" <td>0.650000</td>\n",
" <td>0.175216</td>\n",
" <td>0.683333</td>\n",
" <td>0.633333</td>\n",
" <td>0.550000</td>\n",
" <td>0.400000</td>\n",
" <td>0.633333</td>\n",
" <td>0.400000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>disc-acc</th>\n",
" <td>-0.483333</td>\n",
" <td>1.000000</td>\n",
" <td>-0.216667</td>\n",
" <td>-0.316667</td>\n",
" <td>0.116667</td>\n",
" <td>-0.316667</td>\n",
" <td>0.350000</td>\n",
" <td>-0.157684</td>\n",
" <td>-0.550000</td>\n",
" <td>-0.600000</td>\n",
" <td>-0.466667</td>\n",
" <td>-0.350000</td>\n",
" <td>-0.400000</td>\n",
" <td>0.200000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>disc-zc(nwot)</th>\n",
" <td>0.033333</td>\n",
" <td>0.116667</td>\n",
" <td>0.400000</td>\n",
" <td>0.216667</td>\n",
" <td>1.000000</td>\n",
" <td>0.100000</td>\n",
" <td>0.500000</td>\n",
" <td>0.077814</td>\n",
" <td>-0.050000</td>\n",
" <td>-0.083333</td>\n",
" <td>0.066667</td>\n",
" <td>0.166667</td>\n",
" <td>0.266667</td>\n",
" <td>0.633333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>disc-zc(synflow)</th>\n",
" <td>0.916667</td>\n",
" <td>-0.600000</td>\n",
" <td>0.633333</td>\n",
" <td>0.716667</td>\n",
" <td>-0.083333</td>\n",
" <td>0.600000</td>\n",
" <td>0.150000</td>\n",
" <td>0.317100</td>\n",
" <td>0.983333</td>\n",
" <td>1.000000</td>\n",
" <td>0.733333</td>\n",
" <td>0.333333</td>\n",
" <td>0.516667</td>\n",
" <td>-0.150000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>tenas</th>\n",
" <td>0.300433</td>\n",
" <td>-0.157684</td>\n",
" <td>0.175216</td>\n",
" <td>0.243615</td>\n",
" <td>0.077814</td>\n",
" <td>0.435065</td>\n",
" <td>0.135065</td>\n",
" <td>1.000000</td>\n",
" <td>0.283766</td>\n",
" <td>0.317100</td>\n",
" <td>0.099134</td>\n",
" <td>0.225216</td>\n",
" <td>0.083766</td>\n",
" <td>-0.108550</td>\n",
" </tr>\n",
" <tr>\n",
" <th>zc-disc-post</th>\n",
" <td>-0.033333</td>\n",
" <td>0.200000</td>\n",
" <td>0.400000</td>\n",
" <td>-0.066667</td>\n",
" <td>0.633333</td>\n",
" <td>-0.250000</td>\n",
" <td>0.400000</td>\n",
" <td>-0.108550</td>\n",
" <td>-0.133333</td>\n",
" <td>-0.150000</td>\n",
" <td>-0.100000</td>\n",
" <td>0.100000</td>\n",
" <td>0.216667</td>\n",
" <td>1.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>zc-pt(nwot)</th>\n",
" <td>0.833333</td>\n",
" <td>-0.316667</td>\n",
" <td>0.583333</td>\n",
" <td>1.000000</td>\n",
" <td>0.216667</td>\n",
" <td>0.766667</td>\n",
" <td>0.450000</td>\n",
" <td>0.243615</td>\n",
" <td>0.750000</td>\n",
" <td>0.716667</td>\n",
" <td>0.783333</td>\n",
" <td>0.533333</td>\n",
" <td>0.800000</td>\n",
" <td>-0.066667</td>\n",
" </tr>\n",
" <tr>\n",
" <th>zc-pt(synflow)</th>\n",
" <td>0.933333</td>\n",
" <td>-0.550000</td>\n",
" <td>0.683333</td>\n",
" <td>0.750000</td>\n",
" <td>-0.050000</td>\n",
" <td>0.633333</td>\n",
" <td>0.216667</td>\n",
" <td>0.283766</td>\n",
" <td>1.000000</td>\n",
" <td>0.983333</td>\n",
" <td>0.750000</td>\n",
" <td>0.383333</td>\n",
" <td>0.583333</td>\n",
" <td>-0.133333</td>\n",
" </tr>\n",
" <tr>\n",
" <th>zc-pt-post</th>\n",
" <td>0.683333</td>\n",
" <td>-0.400000</td>\n",
" <td>0.633333</td>\n",
" <td>0.800000</td>\n",
" <td>0.266667</td>\n",
" <td>0.583333</td>\n",
" <td>0.400000</td>\n",
" <td>0.083766</td>\n",
" <td>0.583333</td>\n",
" <td>0.516667</td>\n",
" <td>0.583333</td>\n",
" <td>0.650000</td>\n",
" <td>1.000000</td>\n",
" <td>0.216667</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" avg-acc disc-acc darts-pt zc-pt(nwot) disc-zc(nwot) \\\n",
"avg-acc 1.000000 -0.483333 0.666667 0.833333 0.033333 \n",
"best-acc 0.650000 -0.316667 0.416667 0.766667 0.100000 \n",
"best_zc(nwot) 0.816667 -0.466667 0.550000 0.783333 0.066667 \n",
"best_zc(synflow) 0.416667 -0.350000 0.400000 0.533333 0.166667 \n",
"darts 0.316667 0.350000 0.650000 0.450000 0.500000 \n",
"darts-pt 0.666667 -0.216667 1.000000 0.583333 0.400000 \n",
"disc-acc -0.483333 1.000000 -0.216667 -0.316667 0.116667 \n",
"disc-zc(nwot) 0.033333 0.116667 0.400000 0.216667 1.000000 \n",
"disc-zc(synflow) 0.916667 -0.600000 0.633333 0.716667 -0.083333 \n",
"tenas 0.300433 -0.157684 0.175216 0.243615 0.077814 \n",
"zc-disc-post -0.033333 0.200000 0.400000 -0.066667 0.633333 \n",
"zc-pt(nwot) 0.833333 -0.316667 0.583333 1.000000 0.216667 \n",
"zc-pt(synflow) 0.933333 -0.550000 0.683333 0.750000 -0.050000 \n",
"zc-pt-post 0.683333 -0.400000 0.633333 0.800000 0.266667 \n",
"\n",
" best-acc darts tenas zc-pt(synflow) \\\n",
"avg-acc 0.650000 0.316667 0.300433 0.933333 \n",
"best-acc 1.000000 0.200000 0.435065 0.633333 \n",
"best_zc(nwot) 0.533333 0.216667 0.099134 0.750000 \n",
"best_zc(synflow) 0.250000 0.366667 0.225216 0.383333 \n",
"darts 0.200000 1.000000 0.135065 0.216667 \n",
"darts-pt 0.416667 0.650000 0.175216 0.683333 \n",
"disc-acc -0.316667 0.350000 -0.157684 -0.550000 \n",
"disc-zc(nwot) 0.100000 0.500000 0.077814 -0.050000 \n",
"disc-zc(synflow) 0.600000 0.150000 0.317100 0.983333 \n",
"tenas 0.435065 0.135065 1.000000 0.283766 \n",
"zc-disc-post -0.250000 0.400000 -0.108550 -0.133333 \n",
"zc-pt(nwot) 0.766667 0.450000 0.243615 0.750000 \n",
"zc-pt(synflow) 0.633333 0.216667 0.283766 1.000000 \n",
"zc-pt-post 0.583333 0.400000 0.083766 0.583333 \n",
"\n",
" disc-zc(synflow) best_zc(nwot) best_zc(synflow) \\\n",
"avg-acc 0.916667 0.816667 0.416667 \n",
"best-acc 0.600000 0.533333 0.250000 \n",
"best_zc(nwot) 0.733333 1.000000 0.466667 \n",
"best_zc(synflow) 0.333333 0.466667 1.000000 \n",
"darts 0.150000 0.216667 0.366667 \n",
"darts-pt 0.633333 0.550000 0.400000 \n",
"disc-acc -0.600000 -0.466667 -0.350000 \n",
"disc-zc(nwot) -0.083333 0.066667 0.166667 \n",
"disc-zc(synflow) 1.000000 0.733333 0.333333 \n",
"tenas 0.317100 0.099134 0.225216 \n",
"zc-disc-post -0.150000 -0.100000 0.100000 \n",
"zc-pt(nwot) 0.716667 0.783333 0.533333 \n",
"zc-pt(synflow) 0.983333 0.750000 0.383333 \n",
"zc-pt-post 0.516667 0.583333 0.650000 \n",
"\n",
" zc-pt-post zc-disc-post \n",
"avg-acc 0.683333 -0.033333 \n",
"best-acc 0.583333 -0.250000 \n",
"best_zc(nwot) 0.583333 -0.100000 \n",
"best_zc(synflow) 0.650000 0.100000 \n",
"darts 0.400000 0.400000 \n",
"darts-pt 0.633333 0.400000 \n",
"disc-acc -0.400000 0.200000 \n",
"disc-zc(nwot) 0.266667 0.633333 \n",
"disc-zc(synflow) 0.516667 -0.150000 \n",
"tenas 0.083766 -0.108550 \n",
"zc-disc-post 0.216667 1.000000 \n",
"zc-pt(nwot) 0.800000 -0.066667 \n",
"zc-pt(synflow) 0.583333 -0.133333 \n",
"zc-pt-post 1.000000 0.216667 "
]
},
"execution_count": 138,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"all_corr = []\n",
"for e in range(6):\n",
" d = {}\n",
" for k,v in sq_op_strengths.items():\n",
" d[k] = v[e]\n",
" print(d)\n",
" df = pd.DataFrame(d)\n",
" all_corr.append(df.corr(method='spearman'))\n",
"df = pd.concat(all_corr)\n",
"df_mean = df.groupby(level=0).mean()\n",
"df_mean"
]
},
{
"cell_type": "code",
"execution_count": 149,
"id": "4c7c0062",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" best-acc avg-acc disc-acc best_zc(nwot) \\\n",
"best-acc 1.000000 0.650000 -0.316667 0.533333 \n",
"avg-acc 0.650000 1.000000 -0.483333 0.816667 \n",
"disc-acc -0.316667 -0.483333 1.000000 -0.466667 \n",
"best_zc(nwot) 0.533333 0.816667 -0.466667 1.000000 \n",
"best_zc(synflow) 0.250000 0.416667 -0.350000 0.466667 \n",
"disc-zc(synflow) 0.600000 0.916667 -0.600000 0.733333 \n",
"zc-pt(synflow) 0.633333 0.933333 -0.550000 0.750000 \n",
"disc-zc(nwot) 0.100000 0.033333 0.116667 0.066667 \n",
"zc-pt(nwot) 0.766667 0.833333 -0.316667 0.783333 \n",
"\n",
" best_zc(synflow) disc-zc(synflow) zc-pt(synflow) \\\n",
"best-acc 0.250000 0.600000 0.633333 \n",
"avg-acc 0.416667 0.916667 0.933333 \n",
"disc-acc -0.350000 -0.600000 -0.550000 \n",
"best_zc(nwot) 0.466667 0.733333 0.750000 \n",
"best_zc(synflow) 1.000000 0.333333 0.383333 \n",
"disc-zc(synflow) 0.333333 1.000000 0.983333 \n",
"zc-pt(synflow) 0.383333 0.983333 1.000000 \n",
"disc-zc(nwot) 0.166667 -0.083333 -0.050000 \n",
"zc-pt(nwot) 0.533333 0.716667 0.750000 \n",
"\n",
" disc-zc(nwot) zc-pt(nwot) \n",
"best-acc 0.100000 0.766667 \n",
"avg-acc 0.033333 0.833333 \n",
"disc-acc 0.116667 -0.316667 \n",
"best_zc(nwot) 0.066667 0.783333 \n",
"best_zc(synflow) 0.166667 0.533333 \n",
"disc-zc(synflow) -0.083333 0.716667 \n",
"zc-pt(synflow) -0.050000 0.750000 \n",
"disc-zc(nwot) 1.000000 0.216667 \n",
"zc-pt(nwot) 0.216667 1.000000 \n",
"(5, 8)\n",
"(5, 8)\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAy8AAAH5CAYAAACI1QErAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAADxmklEQVR4nOzdd3hTVR8H8G/SJt17l24KhW72KrKlIiBTNggoWxBEWS4UVBTFF5ClyFRUkA2CLBmFllVWS6HQvfduOtK8fwQCIV1gSxP4fp6nj+Tcc27Ouaa3+d2zBDKZTAYiIiIiIiI1J2zoChAREREREdUGgxciIiIiItIIDF6IiIiIiEgjMHghIiIiIiKNwOCFiIiIiIg0AoMXIiIiIiLSCAxeiIiIiIhII2g3dAVIM9xYOruhq6C2fBetAAAcuhTewDVRX6+38UTEuq8buhpqq9mU+QCA5ItnGrgm6suu7SsI+25RQ1dDrXm9vxQ3v/6goauhtnzmfwsAyI6LbuCaqC8zJ1dcnj2yoauhtlqv+K2hq0BgzwsREREREWkIBi9ERERERKQRGLwQEREREZFGYPBCREREREQagcELERERERFpBAYvRERERESkERi8EBERERGRRmDwQkREREREGoHBCxERERERaQQGL0REREREpBEYvBARERERkUZg8EJERERERBqBwQsREREREWkEBi9ERERERKQRGLwQEREREZFGYPBCREREREQagcELERERERFpBAYvRERERESkERi8EBERERGRRmDwQkREREREGoHBCxERERERaQQGL0REREREpBEYvBARERERkUZg8EJERERERBqBwQsREREREWkEBi9ERERERKQRGLwQEREREZFG0G7oCqib+fPnIzExEdu2bWvoqrzwxKYWsH91IAyc3SGTliMvMhzJx/dCWlxUc2GhEFbtusHMtw3EphaoKClGUXI8Ynf+AlmFFADg0HcEzP3aqhRNPPoXMi+fq+vm1Kur58/i+P6dyEhNgam5JboE9kOnXq9VW0ZSXIzfN6xCQkwU8nOzIRKLYevghB79BqO5X0tFvvioezh/8iiiIsKRk5UBQ2MTePj4I3DwCBibmtV30/4TkZEJbDr1gn4jJ8ikUhTE3kPq+ROoKJFUW86ydQAsWweopGdeC0F68CnFa6t2XWHg5AaRoTEEQiFK83KQE34NOWFX67wt9SEpLR2rtv2O0Nt3INLWQscWfpg28k2YGBlWWUZaUYGdfx/DhWs3EJuYjLLycjjZ2WLY673RtW0rpbxfrf8FR89dUDnHzLEjMKhX9zpvz/MiMjGDXbe+0Hd0lX+uoiKQ8u9hSCXFNRcWCmHZOgCmXi0hMjZDRakExSmJiN/3q+LepElEJuaw7/UGDJwaQyaVIv9eOJJPHIBUUsv7dNsuMPVpDbGJOSpKJChKjkfc7i2Ka2HbtQ+M3JpBZGwKCIUozclC9rVgZF69AEBWr22rT4nJyfh+zTpcvXYdIpE2Atq3x6wpk2BibFxtuWu3bmH/4SMIv3MXsfHx8Pfxxtrvvn1Ota4fYgtrOA0cCyN3T8jKy5ETfhXxe7dDWlRQbbnWK36r8tjtHz5BYew9iM0s4fvJyirzXf90Gsrycp616qSmGLw0AA8PD3z11VcYNGhQQ1elwQjFOnAbPQ3lRYWI27MVQpEYdt37wuXNt3F/S9U3ooecB70FAwcXpJ47BklaErT09GHo2hQQCoHHviCU5mUj7q8tSmVLczPrvD316eblEGxf8z069+6LgWPfQVREOHZv/RkCoQAdewRWWU5aXgZtkQi9BgyBuaU1SktKEHzqGH5evgQT5iyAV4s2AIDQ4HNIS0pEl9f6w9q+EbLT03Bk9++4e+s6PvjqB+jo6j2vpj4VoUgMp/4jUV5chMRj+yAUiWDVriscXhuCuL3baywvq6hA7BP5yp/4YyoUiZAbcQOlOZmQVVTAwNEVNgG9IDIyUQpy1FFhcTFmf7kcJsZG+HTGJEhKSrD+97+w8PvVWP3JPAgEgkrLlZSWYvv+w+jduQOG9XkVYm0RTl++gs9WrcOssSMxsFc3pfxW5mb47N3JSml2Vlb11q76JhTrwOXNiZAWFSHh4B8QikSweaU3nAaOQfSODTWWd+w3Avr2TkgP/heS9GRo6erD0NkdEAqAiufQgDokFOvAbeQUlBcVIH7fdghFYth2fR3OQ8YjavuPNZZ3GjAGBo1ckHb+OCRpyfL7tEsTpWshFOkg68ZFlGSmQ1ZRASPXprDr9QZEJmZIOXWonltYPwoLCzF97jyYmphgyUcLUCwpwY8/b8Tcjz/Fhh++r/J3DwCuhF7HjbBweHo0RXl52XOsdf0Q6ujBY9oilBfmI2rrSgjFOnDoOwJN3p6LiJWfVVv29g+fqKQ5DhgDHQtrFMZHAQDK8nIqzec29l2UFxUwcHlBMXihBmHRsiO0DYxxb8tKlOfnApDfhNzfmgWjJl7IjwyrsqyZTxsYN/FE5C/fQ5KapEjPu3NTJa+svBxFSbF134Dn6PDOX9HcrxUGjpkIAGji6YPc7Ewc+et3tOvaC1paWpWWMzAyxuhps5XSmvu3xJLZU3D57ClF8NK970AYGps8lgmwdXTGio/n4sbFC2jzino+QTf19IeWvgFi925DeaE86CgvyIfzwDEwdHZHQey9Gs8hSUuq9njquWNKr4sSY6FtYAQTDx+1D14OnDyDrNw8rP5kPqzM5T1oVmZmmPHFMlwIvYGOLf0qLacjFmPH91/CyMBAkdbaxxMZWTn44/BRleBFpK0NL/fG9deQ58zMtw209Y0QvWMDygvyAABlBXlwGzEZhm7NUBAVUWVZE88WMHLzQNT2tZCkJyvS8++F13u964O5f3toGxjh/rbVj65Ffi4aj5kBI/fmyL93u8qypt6tYOzeHPc2r1T6Pcu7e0spX9KxPUqvC2MjITIygZlPa40NXvYcOozM7Gxs+N/3sLa0BABYW1pi0ntzcC44BJ07tK+y7PhRIzBxzCgAwHsLFqGktPS51Lm+WHXsAZGxKSJWfoay3GwAQGlOFprPWgwTr5bIraYXu/CJe7iWrj70Gzkj/cJJoEIe/cqk5Sr5dK3toWNuhbQzR+u2MaQ2OOelCtu3b0eXLl3g6+uLt99+G8nJj/4QBQUFYfjw4fD19UVAQADmz5+PrKwsxfHIyEhMnDgRrVu3hp+fHwIDA7F3714AQPfu8i+CCxYsgIeHBzw8PJCQkFBlPQ4ePIhhw4ahdevWaNu2LcaOHYtbt5Rv/uXl5Vi9ejV69uwJb29vdO7cGV988YXieGFhIZYuXYouXbrA29sb3bt3x7p16+riMj0zI3dPFMbdVwQuAFCUGIPSnEwYN/Gqtqx5yw4oiL2vFLi8qLLS05CaGI+WHTsrpbfs+AoK8nIRdz/yqc4nFGpBV08fQuGjgEcpcHmgkbMLBAIBcnOyn63iz4GhszuKk+MVgQsAFKcmojQvR/6ku55USCSKP5zq7MK1G/D1aKIIXADAu6k7bC0tcD70epXltIRCpcDloaYuTsjMya2kxIvFyM0DRYkxii/rAFCcFIfS3CwYNW5WbVlzvzYojI9WClw0mZF7cxTGRyldi6LEWJTmZMHY3bPasub+7VEYF1XjA4LKlEsKIdOA37GqBIVchL+PtyJwAQBfL0/Y2drgXHBItWWFwhfra5mppz8K7kcoAhcAKIyJRElmGky9WlZTUpWZfzsIRWJkXQmqNp9F6wDIpFJkhZ5/pjqT+mPPSyXCwsIgFouxbt06SCQSLF68GDNmzMCuXbsQHByMadOmYe7cufj666+Rn5+Pb775BtOmTcOOHTsgEAgwZ84cNG3aFL///jt0dHQQHR0NqVQ+lGnXrl3o0KEDFi5ciD59+gAAzM3Nq6xLSUkJJk+ejMaNG0MikWDjxo2YOHEijh49ClNTUwDAokWLcObMGcybNw8tW7ZEVlYWrl27BgCQyWSYMmUKkpKS8PHHH8PDwwMpKSmIjo6u12tYEx0LG+SEqz5xkWSkQtfCpuqCQiH07ByRdfUC7Hq+ATPfNhCKdFCUGIPk4/tQnKI
"text/plain": [
"<Figure size 936x936 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from string import ascii_letters\n",
"import numpy as np\n",
"import pandas as pd\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"from pandas import DataFrame\n",
"import pathlib\n",
"\n",
"sns.set_theme(style=\"white\")\n",
"#plt.figure(figsize=(16,10))\n",
"#if 'zc' in flat_op_strengths:\n",
"# flat_op_strengths.pop('zc')\n",
"df = df_mean\n",
"# df=df.reindex([\"best-acc\", \"avg-acc\", \"disc-acc\", \"darts\", \"darts-pt\", 'tenas','disc-zc(synflow)','zc-pt(synflow)', \"disc-zc(nwot)\", \"zc-pt(nwot)\"])\n",
"# df = df[[\"best-acc\", \"avg-acc\", \"disc-acc\", \"darts\", \"darts-pt\",'tenas' ,'disc-zc(synflow)','zc-pt(synflow)', \"disc-zc(nwot)\", \"zc-pt(nwot)\"]]\n",
"#print(df)\n",
"# df=df.reindex([\"best-acc\", \"avg-acc\", \"disc-acc\", 'best_zc(nwot)', \"disc-zc(nwot)\", \"zc-pt(nwot)\"])\n",
"# df = df[[\"best-acc\", \"avg-acc\", \"disc-acc\",'best_zc(nwot)' , \"disc-zc(nwot)\", \"zc-pt(nwot)\"]]\n",
"# df=df.reindex([\"best-acc\", \"avg-acc\", \"disc-acc\", 'best_zc(synflow)','disc-zc(synflow)','zc-pt(synflow)'])\n",
"# df = df[[\"best-acc\", \"avg-acc\", \"disc-acc\", 'best_zc(synflow)','disc-zc(synflow)','zc-pt(synflow)']]\n",
"df=df.reindex([\"best-acc\", \"avg-acc\", \"disc-acc\", 'best_zc(nwot)', 'best_zc(synflow)', 'disc-zc(synflow)','zc-pt(synflow)',\"disc-zc(nwot)\", \"zc-pt(nwot)\"])\n",
"df = df[[\"best-acc\", \"avg-acc\", \"disc-acc\",'best_zc(nwot)' , 'best_zc(synflow)', 'disc-zc(synflow)','zc-pt(synflow)',\"disc-zc(nwot)\", \"zc-pt(nwot)\"]]\n",
"\n",
"\n",
"# Compute the correlation matrix\n",
"#corr = df.corr(method='spearman')\n",
"corr=df\n",
"print(corr)\n",
"\n",
"# Generate a mask for the upper triangle\n",
"mask = np.tril(np.ones_like(corr, dtype=bool))\n",
"\n",
"for i in range(5,9):\n",
" for j in range(0,9):\n",
" mask[i][j]=True\n",
"\n",
"masked_corr = corr.loc[~np.all(mask, axis=1), ~np.all(mask, axis=0)]\n",
"masked_mask = mask[0:5,1:9]\n",
"print(np.shape(masked_corr))\n",
"print(np.shape(masked_mask))\n",
"\n",
"\n",
"# Set up the matplotlib figure\n",
"f, ax = plt.subplots(figsize=(13, 13))\n",
"sns.set(font_scale=1.3)\n",
"# Generate a custom diverging colormap\n",
"cmap = sns.diverging_palette(230, 20, as_cmap=True)\n",
"\n",
"# Draw the heatmap with the mask and correct aspect ratio\n",
"g = sns.heatmap(masked_corr, mask=masked_mask, cmap=cmap, center=0,\n",
" square=True, linewidths=.5, cbar_kws={\"shrink\": .267}, annot=True)\n",
"\n",
"g.set_yticklabels(labels=g.get_yticklabels(), va='center', fontsize = 14.5)\n",
"g.set_xticklabels(g.get_xmajorticklabels(), fontsize = 14.5, rotation=70)\n",
"\n",
"f.savefig(pathlib.Path('metric_correlation').with_suffix('.pdf'), bbox_inches='tight')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "abd8548a",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 10,
"id": "b1f23258",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" best-acc avg-acc disc-acc darts-pt disc-zc zc-pt darts\n",
"best-acc 1.000000 0.650000 -0.316667 0.416667 0.100000 0.766667 0.200000\n",
"avg-acc 0.650000 1.000000 -0.483333 0.666667 0.033333 0.833333 0.316667\n",
"disc-acc -0.316667 -0.483333 1.000000 -0.216667 0.116667 -0.316667 0.350000\n",
"darts-pt 0.416667 0.666667 -0.216667 1.000000 0.400000 0.583333 0.650000\n",
"disc-zc 0.100000 0.033333 0.116667 0.400000 1.000000 0.216667 0.500000\n",
"zc-pt 0.766667 0.833333 -0.316667 0.583333 0.216667 1.000000 0.450000\n",
"darts 0.200000 0.316667 0.350000 0.650000 0.500000 0.450000 1.000000\n",
"(6, 3)\n",
"(6, 3)\n"
]
},
{
"data": {
"text/plain": [
"<Figure size 576x360 with 0 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAIPCAYAAACFR+6IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABwoklEQVR4nO3dd1QUVxsH4N/SO9J7E6QIwlpR0RiN+lmTYC/YNRoL9gbGFnsJRrGhJtYYTVQsMdYYI6CigqiIFZCO9A5Lme8P4iabhWUWd4XB9zlnz8neuXfmnc36cvfOnbk8hmEYEEII4SSFhg6AEEJI/VESJ4QQDqMkTgghHEZJnBBCOIySOCGEcBglcUII4TBK4oQQwmFKDXnwiMUTGvLwnNdm448NHQIhpIFRT5wQQjiMkjghhHAYJXFCCOEwSuKEEMJhlMQJIYTDKIkTQgiHURInhBAOoyROCCEcRkmcEEI4jJI4IYRwGCVxQgjhMErihBDCYayTeEFBAXJzc8XKc3NzUVhYKMuYCCGEsMQ6ic+fPx/nz58XK//tt9+wYMECmQZFCCGEHdZJPCoqCp6enmLlHTp0wMOHD2UZEyGEEJZYJ/GSkhIoKiqK70BBAcXFxTINihBCCDusk7iDgwOuXr0qVn758mU0b95cpkERQghhh/XKPpMnT8aiRYuQkZGBTp06AQDCwsJw8uRJbNiwQW4BEkIIqR3rJN6vXz+UlJQgMDAQx44dAwCYmppixYoVGDBggNwCJIQQUjup1tgcPHgwBg8ejOzsbACAvr6+XIIihBDCDusknpiYiIqKCtjZ2Ykk7/j4eCgpKcHS0lIuARJCCKkd6wubfn5+iIiIECuPjIyEv7+/TIMihBDCDuskHhMTgzZt2oiV8/l8PH36VKZBEUIIYYd1Eq+oqIBAIBArFwgEKC8vl2lQhBBC2GGdxF1dXXHmzBmx8lOnTsHZ2VmmQRFCCGGH9YXNGTNmYPLkyYiPjxeZJx4SEoKgoCC5BUgIIaR2rHvinTt3xv79+1FQUICAgAAEBASgsLAQQUFB8PLykmeMhBBCaiHVPPHOnTujc+fO8oqFEEKIlGhRCEII4TCpeuLBwcE4f/48kpOTxWakXL9+XaaBEUIIqRvrnviRI0ewatUqWFtbIykpCV5eXjA3N0deXh49O4UQQhoI65748ePHsXz5cnh7eyM4OBhTpkyBlZUVNm3ahJKSEnnG+F5U9I1gNXAUtOydwVRWIC8mCkkXjqOyuKjuxgqKMPnkfzBo2wUq+oaoLC1BcWIcYo/sAFNZCQCwGToJBu26iDVNDD6KjNv064QQIl+sk3hKSgrat28PAFBRUUFRUXUSHDJkCHx8fODn5yefCN+DgqoaWny1CBVFBYj7aTcUVFRh0Xco7MfPwYtda+ts39xnOjRtHJB2/TxKUhOhqKEJnRauAE8BQKWwniA3G7HHdoq0FWRnyvp0CCFEDOskrqOjI0zcxsbGiI2NhbOzMwoKClBaWiq3AN+HoeenUNbWxYtda1GenwsAKM/LgdN0f+i6eCAvJqrWtvptvKDr7IFnO1ahJDVRWJ4XLf78mKqKchQnxMo8fkIIqQvrJM7n83H//n04OTmhR48eWL9+PR4/fowbN24Ie+iNja6zBwrjXggTOAAUvXmFsuwM6LrwJSZxw46foiD2mUgCJ4SQxoZ1El+0aJFwLc3p06ejoKAAISEhcHJyapRDKQCgZmyGnKi7YuWl6SlQMzavvaGCIjQsbJF5909YDBgBg7ZeUFBRRdGb10j67WeUJL8Rqa7STB/uK3ZAUVUNpRlpeBtyBVn3bsn6dAghRAzrJP7v54Wrqqpi+fLlcglIlhTVNVFZIr6Ic2VJMVT0DWttp6ShCQUlJRi09UJpZhrenDwAgIFZzy/RYvJCRG9egsriQgBAcWoCipPiUZKeDEVVNei37gSbIROhpKGF9Ju/y+vUCCEEgJTzxN8JCgrCiBEjoKOjI+t4ZI6pqZAnuQ1PoXrmJU9REa9/3IaKwnwAQFFiHNwWb4KxV0+kXg0GAGSEiC4enRfzEFBQgOlnn+Nt6FUwFRXvdwKEECJBve7Y3LNnD/Ly8mQdi8xVlhRBSV1DrFxRTUPiFMOK4iIwVVUoSU8WJnAAqCjMR0laEtRMLCQeNyfqLhRV1SQP2RBCiAzUK4kzTI3920an9G1qjYlUzcQcpW9Tam3HVJRDkJMJXk1ddh4PCkrKkg/M+7sdRz4nQgh3Nelnp+Q9i4JWcycoazcTlmlYN4eqvlH1sIcEuU8joWZiIdJWSVsX6qaWKE6Kk9hW38MTlaUlKH2b+h7RE0JI3eo1Jr569WoYGBjIOhaZy7z7J4w6f4bm43yRei0YCsoqsOg7FIVvXolMLzTr+QXMen2JJxsWQJCTBQBIv/k79Ft3gv2EOUi9fq663mefo7K0BBm3/wAAqDQzgM2wSch+eAeCrAwoqKnDoE1nNHNri8RzP4GppPFwQoh81SuJDxw4UNZxyEVVWSleBm2C1eejYDfqazCVlciLeYikC8frbFtRkIeXezfAYsAI2A6fAoBBwetniDu2GxVFBQCAyrJSVJaWwOyzz6GkqQOmqhIlqYmIPboTuY/vy/nsCCEE4DEsB7gDAwNr3gGPB1VVVdja2uKTTz6BiooK64NHLJ7Aui4R12bjjw0dAiGkgbHuiZ8+fRo5OTkoKSkRTi3Mz8+Huro61NXVkZ2dDXNzcxw9ehTm5jQrgxBCPgTWFzYXL14MJycnXLx4EeHh4QgPD8fFixfh4uKClStX4vr16zAwMMDGjRvlGS8hhJB/YZ3EAwIC4Ofnh+bNmwvLmjdvjiVLlmDr1q2wsLDAwoULERkZKZdACSGEiGOdxFNSUqCmpiZWrqKigtTU6ql0FhYWnLgJiBBCmgrWSdzFxQXfffcdCgoKhGUFBQXYtm0bWrZsCQBITEyEsbGx7KMkhBBSI9YXNlesWIGvvvoKn3zyCezs7AAAcXFx0NTUxP79+wEAGRkZGD9+vFwCJYQQIo71FEMAKC0txblz5/D69WsAgIODAwYOHFjjMAsbNMXw/dAUQ0II6564QCCAmpoahg0bJs94CCGESIH1mHiXLl2watUqPHr0SJ7xEEIIkQLrJO7r64snT55g2LBh6NevH/bt24f09HR5xkYIIaQOUo2JA0BsbCyCg4Nx4cIFpKeno2PHjhg0aBD69+8v9cFpTPz90Jg4IUTqJP5vd+7cwcaNG/Hs2TPExMRI3Z6S+PuhJE4IqddTDNPT03H27FkEBwcjNjYWHh4eso6LEEIIC6yTeGlpKa5cuYLg4GDcvXsXxsbG+Pzzz/Hll18K540TQgj5sFgn8c6dO4NhGPTq1Qv79+9Hx44dwePVseIwIYQQuWKdxJctW4Y+ffpAQ0N84WFCCCENg3USHzRoEBISEnDp0iUkJSWhvLxcZPv69etlHhwhhBDJWCfxsLAwTJ06FdbW1oiPj0eLFi2QlJQEAMIHYBFCCPmwWN/s8/3332PcuHH47bffoKKigh07duDPP/8En8/nzJqbhBDS1LBO4q9evcLgwYMBAIqKihAIBNDS0sLs2bMRFBQktwAJIYTUjnUSV1ZWFs5GMTAwQEpKCgBAU1MTb9++lU90hBBCJGI9Ju7o6IiYmBjY2tqidevW2L17NyorK/HLL7/A3t5enjESQgipBeue+LRp06CsrAwAmDVrFvLz8zFt2jQ8fPgQ/v7+cguQEEJI7aS62ecdCwsLXLhwAbm5udDV1aWbfgghpIHU69kp7zRr1kxGYRBCCKkP1sMphBBCGh9K4oQQwmGUxAkhhMMoiRNCCIdREieEEA6jJE4IIRxGSZwQQjiMkjghhHAYJXFCCOEwSuKEEMJhPIZhmIYOghBCSP2817NT3teZ248b8vCc592pFQBg05mbDRwJty3y7tbQIRBSbzScQgghHEZJnBBCOIySOCGEcBglcUII4TBK4oQQwmGUxAkhhMMoiRNCCIdREieEEA6jJE4IIRxGSZwQQjiMkjghhHAYJXFCCOEw1km8oKAAubm5YuW5ubkoLCyUZUyEEEJYYp3E58+
"text/plain": [
"<Figure size 792x648 with 2 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from string import ascii_letters\n",
"import numpy as np\n",
"import pandas as pd\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"from pandas import DataFrame\n",
"import pathlib\n",
"\n",
"sns.set_theme(style=\"white\")\n",
"plt.figure(figsize=(8,5))\n",
"#if 'zc' in flat_op_strengths:\n",
"# flat_op_strengths.pop('zc')\n",
"df = df_mean\n",
"df=df.reindex([\"best-acc\", \"avg-acc\", \"disc-acc\", \"darts-pt\", \"disc-zc\", \"zc-pt\", \"darts\"])\n",
"df = df[[\"best-acc\", \"avg-acc\", \"disc-acc\", \"darts-pt\", \"disc-zc\", \"zc-pt\", \"darts\"]]\n",
"#print(df)\n",
"\n",
"# Compute the correlation matrix\n",
"#corr = df.corr(method='spearman')\n",
"corr = df\n",
"print(corr)\n",
"\n",
"# Generate a mask for the upper triangle\n",
"mask = np.triu(np.ones_like(corr, dtype=bool))\n",
"\n",
"for i in range(3,7):\n",
" for j in range(0,7):\n",
" mask[j][i]=True\n",
"\n",
"masked_corr = df.loc[~np.all(mask, axis=1), ~np.all(mask, axis=0)]\n",
"masked_mask = mask[1:7,0:3]\n",
"print(np.shape(masked_corr))\n",
"print(np.shape(masked_mask))\n",
"\n",
"\n",
"# Set up the matplotlib figure\n",
"f, ax = plt.subplots(figsize=(11, 9))\n",
"sns.set(font_scale=1.3)\n",
"# Generate a custom diverging colormap\n",
"cmap = sns.diverging_palette(230, 20, as_cmap=True)\n",
"\n",
"# Draw the heatmap with the mask and correct aspect ratio\n",
"g = sns.heatmap(masked_corr, mask=masked_mask, cmap=cmap, center=0,\n",
" square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, annot=True)\n",
"\n",
"g.set_yticklabels(labels=g.get_yticklabels(), va='center', fontsize = 14.5)\n",
"g.set_xticklabels(g.get_xmajorticklabels(), fontsize = 14.5)\n",
"\n",
"f.savefig(pathlib.Path('metric_correlation_vert').with_suffix('.pdf'), bbox_inches='tight')"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "82964512",
"metadata": {},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'acc_d' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-35-c4f853d0b0d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbest_archs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0macc_d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbop\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'acc_d' is not defined"
]
}
],
"source": [
"plot_data = []\n",
"metrics = op_strengths[0].keys()\n",
"best_archs = {}\n",
"for m in metrics:\n",
" best_archs[m] = []\n",
" plot_data.append([m,{}])\n",
"for j,edge in enumerate(op_strengths):\n",
" bop = -1\n",
" bm = -1\n",
" for i,m in enumerate(metrics):\n",
" if 'pt' in m:\n",
" bm = min(edge[m])\n",
" else:\n",
" bm = max(edge[m])\n",
" bop = edge[m].index(bm)\n",
" #print(m,ops[bop],bm)\n",
" best_archs[m].append(bop)\n",
" \n",
"i=0\n",
"for k,v in best_archs.items():\n",
"\n",
" acc = stat.mean(acc_d[tuple(v)])\n",
" \n",
" for j,bop in enumerate(v):\n",
" plot_data[i][1][j]=ops[bop]\n",
" plot_data[i][1]['acc']=acc\n",
" \n",
" i+=1\n",
" print(k,v,acc)\n",
"print(ops)"
]
},
{
"cell_type": "code",
"execution_count": 477,
"id": "d7221f6c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[['avg-acc',\n",
" {0: 'nor_conv_3x3',\n",
" 1: 'nor_conv_3x3',\n",
" 2: 'nor_conv_3x3',\n",
" 3: 'nor_conv_3x3',\n",
" 4: 'nor_conv_3x3',\n",
" 5: 'nor_conv_3x3',\n",
" 'acc': 93.76}],\n",
" ['disc-acc',\n",
" {0: 'none',\n",
" 1: 'none',\n",
" 2: 'none',\n",
" 3: 'skip_connect',\n",
" 4: 'none',\n",
" 5: 'none',\n",
" 'acc': 86.45333333333333}],\n",
" ['darts-pt',\n",
" {0: 'skip_connect',\n",
" 1: 'skip_connect',\n",
" 2: 'nor_conv_3x3',\n",
" 3: 'skip_connect',\n",
" 4: 'skip_connect',\n",
" 5: 'skip_connect',\n",
" 'acc': 80.57}],\n",
" ['zc-pt',\n",
" {0: 'nor_conv_3x3',\n",
" 1: 'nor_conv_3x3',\n",
" 2: 'nor_conv_1x1',\n",
" 3: 'skip_connect',\n",
" 4: 'nor_conv_3x3',\n",
" 5: 'nor_conv_1x1',\n",
" 'acc': 94.195}],\n",
" ['disc-zc',\n",
" {0: 'skip_connect',\n",
" 1: 'skip_connect',\n",
" 2: 'nor_conv_1x1',\n",
" 3: 'skip_connect',\n",
" 4: 'skip_connect',\n",
" 5: 'skip_connect',\n",
" 'acc': 77.04333333333334}],\n",
" ['best-acc',\n",
" {0: 'nor_conv_3x3',\n",
" 1: 'nor_conv_3x3',\n",
" 2: 'nor_conv_3x3',\n",
" 3: 'skip_connect',\n",
" 4: 'nor_conv_3x3',\n",
" 5: 'nor_conv_1x1',\n",
" 'acc': 94.37333333333333}],\n",
" ['darts',\n",
" {0: 'skip_connect',\n",
" 1: 'skip_connect',\n",
" 2: 'skip_connect',\n",
" 3: 'skip_connect',\n",
" 4: 'skip_connect',\n",
" 5: 'skip_connect',\n",
" 'acc': 54.29666666666667}]]"
]
},
"execution_count": 477,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"plot_data"
]
},
{
"cell_type": "code",
"execution_count": 478,
"id": "dbeb6e7c",
"metadata": {},
"outputs": [
{
"data": {
"application/javascript": [
"\n",
"/*!\n",
" * Copyright (c) Facebook, Inc. and its affiliates.\n",
" * \n",
" * This source code is licensed under the MIT license found in the\n",
" * LICENSE file in the root directory of this source tree.\n",
" */\n",
"var hiplot=function(t){var e={};function r(n){if(e[n])return e[n].exports;var i=e[n]={i:n,l:!1,exports:{}};return t[n].call(i.exports,i,i.exports,r),i.l=!0,i.exports}return r.m=t,r.c=e,r.d=function(t,e,n){r.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:n})},r.r=function(t){\"undefined\"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:\"Module\"}),Object.defineProperty(t,\"__esModule\",{value:!0})},r.t=function(t,e){if(1&e&&(t=r(t)),8&e)return t;if(4&e&&\"object\"==typeof t&&t&&t.__esModule)return t;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,\"default\",{enumerable:!0,value:t}),2&e&&\"string\"!=typeof t)for(var i in t)r.d(n,i,function(e){return t[e]}.bind(null,i));return n},r.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return r.d(e,\"a\",e),e},r.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},r.p=\"\",r(r.s=62)}([function(t,e,r){\"use strict\";t.exports=r(43)},function(t,e,r){\"use strict\";r.r(e),r.d(e,\"version\",(function(){return n})),r.d(e,\"bisect\",(function(){return s})),r.d(e,\"bisectRight\",(function(){return l})),r.d(e,\"bisectLeft\",(function(){return h})),r.d(e,\"ascending\",(function(){return i})),r.d(e,\"bisector\",(function(){return o})),r.d(e,\"cross\",(function(){return p})),r.d(e,\"descending\",(function(){return u})),r.d(e,\"deviation\",(function(){return g})),r.d(e,\"extent\",(function(){return b})),r.d(e,\"histogram\",(function(){return A})),r.d(e,\"thresholdFreedmanDiaconis\",(function(){return z})),r.d(e,\"thresholdScott\",(function(){return L})),r.d(e,\"thresholdSturges\",(function(){return N})),r.d(e,\"max\",(function(){return I})),r.d(e,\"mean\",(function(){return O})),r.d(e,\"median\",(function(){return P})),r.d(e,\"merge\",(function(){return R})),r.d(e,\"min\",(function(){return F})),r.d(e,\"pairs\",(function(){return d})),r.d(e,\"permute\",(function(){return U})),r.d(e,\"quantile\",(function(){return j})),r.d(e,\"range\",(function(){return w})),r.d(e,\"scan\",(function(){return H})),r.d(e,\"shuffle\",(function(){return B})),r.d(e,\"sum\",(function(){return W})),r.d(e,\"ticks\",(function(){return T})),r.d(e,\"tickIncrement\",(function(){return E})),r.d(e,\"tickStep\",(function(){return D})),r.d(e,\"transpose\",(function(){return Y})),r.d(e,\"variance\",(function(){return f})),r.d(e,\"zip\",(function(){return G})),r.d(e,\"axisTop\",(function(){return et})),r.d(e,\"axisRight\",(function(){return rt})),r.d(e,\"axisBottom\",(function(){return nt})),r.d(e,\"axisLeft\",(function(){return it})),r.d(e,\"brush\",(function(){return Ci})),r.d(e,\"brushX\",(function(){return Si})),r.d(e,\"brushY\",(function(){return Mi})),r.d(e,\"brushSelection\",(function(){return wi})),r.d(e,\"chord\",(function(){return Ii})),r.d(e,\"ribbon\",(function(){return Xi})),r.d(e,\"nest\",(function(){return Ji})),r.d(e,\"set\",(function(){return lo})),r.d(e,\"map\",(function(){return Ki})),r.d(e,\"keys\",(function(){return ho})),r.d(e,\"values\",(function(){return so})),r.d(e,\"entries\",(function(){return co})),r.d(e,\"color\",(function(){return Ye})),r.d(e,\"rgb\",(function(){return Qe})),r.d(e,\"hsl\",(function(){return er})),r.d(e,\"lab\",(function(){return go})),r.d(e,\"hcl\",(function(){return So})),r.d(e,\"lch\",(function(){return wo})),r.d(e,\"gray\",(function(){return fo})),r.d(e,\"cubehelix\",(function(){return jo})),r.d(e,\"contours\",(function(){return Bo})),r.d(e,\"contourDensity\",(function(){return Qo})),r.d(e,\"dispatch\",(function(){return ct})),r.d(e,\"drag\",(function(){return ea})),r.d(e,\"dragDisable\",(function(){return Ce})),r.d(e,\"dragEnable\",(function(){return Te})),r.d(e,\"dsvFormat\",(function(){return ha})),r.d(e,\"csvParse\",(function(){return da})),r.d(e,\"csvParseRows\",(function(){return ca})),r.d(e,\"csvFormat\",(function(){return pa})),r.d(e,\"csvFormatBody\",(function(){return ua})),r.d(e,\"csvFormatRows\",(function(){return ma})),r.d(e,\"csvFormatRow\",(function(){return fa})),r.d(e,\"csvFormatValue\",(function(){return ga})),r.d(e,\"tsvParse\",(function(){retu
"/*!\n",
" * jQuery JavaScript Library v3.5.1\n",
" * https://jquery.com/\n",
" *\n",
" * Includes Sizzle.js\n",
" * https://sizzlejs.com/\n",
" *\n",
" * Copyright JS Foundation and other contributors\n",
" * Released under the MIT license\n",
" * https://jquery.org/license\n",
" *\n",
" * Date: 2020-05-04T22:49Z\n",
" */!function(e,r){\"use strict\";\"object\"==typeof t.exports?t.exports=e.document?r(e,!0):function(t){if(!t.document)throw new Error(\"jQuery requires a window with a document\");return r(t)}:r(e)}(\"undefined\"!=typeof window?window:this,(function(r,i){\"use strict\";var o=[],a=Object.getPrototypeOf,l=o.slice,h=o.flat?function(t){return o.flat.call(t)}:function(t){return o.concat.apply([],t)},s=o.push,d=o.indexOf,c={},p=c.toString,u=c.hasOwnProperty,m=u.toString,f=m.call(Object),g={},b=function(t){return\"function\"==typeof t&&\"number\"!=typeof t.nodeType},_=function(t){return null!=t&&t===t.window},x=r.document,v={type:!0,src:!0,nonce:!0,noModule:!0};function k(t,e,r){var n,i,o=(r=r||x).createElement(\"script\");if(o.text=t,e)for(n in v)(i=e[n]||e.getAttribute&&e.getAttribute(n))&&o.setAttribute(n,i);r.head.appendChild(o).parentNode.removeChild(o)}function y(t){return null==t?t+\"\":\"object\"==typeof t||\"function\"==typeof t?c[p.call(t)]||\"object\":typeof t}var w=function(t,e){return new w.fn.init(t,e)};function S(t){var e=!!t&&\"length\"in t&&t.length,r=y(t);return!b(t)&&!_(t)&&(\"array\"===r||0===e||\"number\"==typeof e&&e>0&&e-1 in t)}w.fn=w.prototype={jquery:\"3.5.1\",constructor:w,length:0,toArray:function(){return l.call(this)},get:function(t){return null==t?l.call(this):t<0?this[t+this.length]:this[t]},pushStack:function(t){var e=w.merge(this.constructor(),t);return e.prevObject=this,e},each:function(t){return w.each(this,t)},map:function(t){return this.pushStack(w.map(this,(function(e,r){return t.call(e,r,e)})))},slice:function(){return this.pushStack(l.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(w.grep(this,(function(t,e){return(e+1)%2})))},odd:function(){return this.pushStack(w.grep(this,(function(t,e){return e%2})))},eq:function(t){var e=this.length,r=+t+(t<0?e:0);return this.pushStack(r>=0&&r<e?[this[r]]:[])},end:function(){return this.prevObject||this.constructor()},push:s,sort:o.sort,splice:o.splice},w.extend=w.fn.extend=function(){var t,e,r,n,i,o,a=arguments[0]||{},l=1,h=arguments.length,s=!1;for(\"boolean\"==typeof a&&(s=a,a=arguments[l]||{},l++),\"object\"==typeof a||b(a)||(a={}),l===h&&(a=this,l--);l<h;l++)if(null!=(t=arguments[l]))for(e in t)n=t[e],\"__proto__\"!==e&&a!==n&&(s&&n&&(w.isPlainObject(n)||(i=Array.isArray(n)))?(r=a[e],o=i&&!Array.isArray(r)?[]:i||w.isPlainObject(r)?r:{},i=!1,a[e]=w.extend(s,o,n)):void 0!==n&&(a[e]=n));return a},w.extend({expando:\"jQuery\"+(\"3.5.1\"+Math.random()).replace(/\\D/g,\"\"),isReady:!0,error:function(t){throw new Error(t)},noop:function(){},isPlainObject:function(t){var e,r;return!(!t||\"[object Object]\"!==p.call(t))&&(!(e=a(t))||\"function\"==typeof(r=u.call(e,\"constructor\")&&e.constructor)&&m.call(r)===f)},isEmptyObject:function(t){var e;for(e in t)return!1;return!0},globalEval:function(t,e,r){k(t,{nonce:e&&e.nonce},r)},each:function(t,e){var r,n=0;if(S(t))for(r=t.length;n<r&&!1!==e.call(t[n],n,t[n]);n++);else for(n in t)if(!1===e.call(t[n],n,t[n]))break;return t},makeArray:function(t,e){var r=e||[];return null!=t&&(S(Object(t))?w.merge(r,\"string\"==typeof t?[t]:t):s.call(r,t)),r},inArray:function(t,e,r){return null==e?-1:d.call(e,t,r)},merge:function(t,e){for(var r=+e.length,n=0,i=t.length;n<r;n++)t[i++]=e[n];return t.length=i,t},grep:function(t,e,r){for(var n=[],i=0,o=t.length,a=!r;i<o;i++)!e(t[i],i)!==a&&n.push(t[i]);return n},map:function(t,e,r){var n,i,o=0,a=[];if(S(t))for(n=t.length;o<n;o++)null!=(i=e(t[o],o,r))&&a.push(i);else for(o in t)null!=(i=e(t[o],o,r))&&a.push(i);return h(a)},guid:1,support:g}),\"function\"==typeof Symbol&&(w.fn[Symbol.iterator]=o[Symbol.iterator]),w.each(\"Boolean Number String Function Array Date RegExp Object Error Symbol\".split(\" \"),(function(t,e){c[\"[object \"+e+\"]\"]=e.toLowerCase()}));var M=\n",
"/*!\n",
" * Sizzle CSS Selector Engine v2.3.5\n",
" * https://sizzlejs.com/\n",
" *\n",
" * Copyright JS Foundation and other contributors\n",
" * Released under the MIT license\n",
" * https://js.foundation/\n",
" *\n",
" * Date: 2020-03-14\n",
" */\n",
"function(t){var e,r,n,i,o,a,l,h,s,d,c,p,u,m,f,g,b,_,x,v=\"sizzle\"+1*new Date,k=t.document,y=0,w=0,S=ht(),M=ht(),C=ht(),T=ht(),E=function(t,e){return t===e&&(c=!0),0},D={}.hasOwnProperty,N=[],A=N.pop,j=N.push,z=N.push,L=N.slice,I=function(t,e){for(var r=0,n=t.length;r<n;r++)if(t[r]===e)return r;return-1},O=\"checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped\",P=\"[\\\\x20\\\\t\\\\r\\\\n\\\\f]\",R=\"(?:\\\\\\\\[\\\\da-fA-F]{1,6}\"+P+\"?|\\\\\\\\[^\\\\r\\\\n\\\\f]|[\\\\w-]|[^\\0-\\\\x7f])+\",F=\"\\\\[\"+P+\"*(\"+R+\")(?:\"+P+\"*([*^$|!~]?=)\"+P+\"*(?:'((?:\\\\\\\\.|[^\\\\\\\\'])*)'|\\\"((?:\\\\\\\\.|[^\\\\\\\\\\\"])*)\\\"|(\"+R+\"))|)\"+P+\"*\\\\]\",U=\":(\"+R+\")(?:\\\\((('((?:\\\\\\\\.|[^\\\\\\\\'])*)'|\\\"((?:\\\\\\\\.|[^\\\\\\\\\\\"])*)\\\")|((?:\\\\\\\\.|[^\\\\\\\\()[\\\\]]|\"+F+\")*)|.*)\\\\)|)\",H=new RegExp(P+\"+\",\"g\"),B=new RegExp(\"^\"+P+\"+|((?:^|[^\\\\\\\\])(?:\\\\\\\\.)*)\"+P+\"+$\",\"g\"),W=new RegExp(\"^\"+P+\"*,\"+P+\"*\"),Y=new RegExp(\"^\"+P+\"*([>+~]|\"+P+\")\"+P+\"*\"),q=new RegExp(P+\"|>\"),G=new RegExp(U),V=new RegExp(\"^\"+R+\"$\"),Q={ID:new RegExp(\"^#(\"+R+\")\"),CLASS:new RegExp(\"^\\\\.(\"+R+\")\"),TAG:new RegExp(\"^(\"+R+\"|[*])\"),ATTR:new RegExp(\"^\"+F),PSEUDO:new RegExp(\"^\"+U),CHILD:new RegExp(\"^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\\\(\"+P+\"*(even|odd|(([+-]|)(\\\\d*)n|)\"+P+\"*(?:([+-]|)\"+P+\"*(\\\\d+)|))\"+P+\"*\\\\)|)\",\"i\"),bool:new RegExp(\"^(?:\"+O+\")$\",\"i\"),needsContext:new RegExp(\"^\"+P+\"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\\\(\"+P+\"*((?:-\\\\d)?\\\\d*)\"+P+\"*\\\\)|)(?=[^-]|$)\",\"i\")},X=/HTML$/i,Z=/^(?:input|select|textarea|button)$/i,$=/^h\\d$/i,K=/^[^{]+\\{\\s*\\[native \\w/,J=/^(?:#([\\w-]+)|(\\w+)|\\.([\\w-]+))$/,tt=/[+~]/,et=new RegExp(\"\\\\\\\\[\\\\da-fA-F]{1,6}\"+P+\"?|\\\\\\\\([^\\\\r\\\\n\\\\f])\",\"g\"),rt=function(t,e){var r=\"0x\"+t.slice(1)-65536;return e||(r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320))},nt=/([\\0-\\x1f\\x7f]|^-?\\d)|^-$|[^\\0-\\x1f\\x7f-\\uFFFF\\w-]/g,it=function(t,e){return e?\"\\0\"===t?\"<22>\":t.slice(0,-1)+\"\\\\\"+t.charCodeAt(t.length-1).toString(16)+\" \":\"\\\\\"+t},ot=function(){p()},at=vt((function(t){return!0===t.disabled&&\"fieldset\"===t.nodeName.toLowerCase()}),{dir:\"parentNode\",next:\"legend\"});try{z.apply(N=L.call(k.childNodes),k.childNodes),N[k.childNodes.length].nodeType}catch(t){z={apply:N.length?function(t,e){j.apply(t,L.call(e))}:function(t,e){for(var r=t.length,n=0;t[r++]=e[n++];);t.length=r-1}}}function lt(t,e,n,i){var o,l,s,d,c,m,b,_=e&&e.ownerDocument,k=e?e.nodeType:9;if(n=n||[],\"string\"!=typeof t||!t||1!==k&&9!==k&&11!==k)return n;if(!i&&(p(e),e=e||u,f)){if(11!==k&&(c=J.exec(t)))if(o=c[1]){if(9===k){if(!(s=e.getElementById(o)))return n;if(s.id===o)return n.push(s),n}else if(_&&(s=_.getElementById(o))&&x(e,s)&&s.id===o)return n.push(s),n}else{if(c[2])return z.apply(n,e.getElementsByTagName(t)),n;if((o=c[3])&&r.getElementsByClassName&&e.getElementsByClassName)return z.apply(n,e.getElementsByClassName(o)),n}if(r.qsa&&!T[t+\" \"]&&(!g||!g.test(t))&&(1!==k||\"object\"!==e.nodeName.toLowerCase())){if(b=t,_=e,1===k&&(q.test(t)||Y.test(t))){for((_=tt.test(t)&&bt(e.parentNode)||e)===e&&r.scope||((d=e.getAttribute(\"id\"))?d=d.replace(nt,it):e.setAttribute(\"id\",d=v)),l=(m=a(t)).length;l--;)m[l]=(d?\"#\"+d:\":scope\")+\" \"+xt(m[l]);b=m.join(\",\")}try{return z.apply(n,_.querySelectorAll(b)),n}catch(e){T(t,!0)}finally{d===v&&e.removeAttribute(\"id\")}}}return h(t.replace(B,\"$1\"),e,n,i)}function ht(){var t=[];return function e(r,i){return t.push(r+\" \")>n.cacheLength&&delete e[t.shift()],e[r+\" \"]=i}}function st(t){return t[v]=!0,t}function dt(t){var e=u.createElement(\"fieldset\");try{return!!t(e)}catch(t){return!1}finally{e.parentNode&&e.parentNode.removeChild(e),e=null}}function ct(t,e){for(var r=t.split(\"|\"),i=r.length;i--;)n.attrHandle[r[i]]=e}function pt(t,e){var r=e&&t,n=r&&1===t.nodeType&&1===e.nodeType&&t.sourceIndex-e.sourceIndex;if(n)return n;if(r)for(;r=r.n
"/*! DataTables 1.10.23\n",
" * ©2008-2020 SpryMedia Ltd - datatables.net/license\n",
" */\n",
"!function(e){\"use strict\";t.exports=function(t,n){return t||(t=window),n||(n=\"undefined\"!=typeof window?r(3):r(3)(t)),e(n,t,t.document)}}((function(t,e,r,n){\"use strict\";var i,o,a,l,h=function(e){this.$=function(t,e){return this.api(!0).$(t,e)},this._=function(t,e){return this.api(!0).rows(t,e).data()},this.api=function(t){return new o(t?ae(this[i.iApiIndex]):this)},this.fnAddData=function(e,r){var i=this.api(!0),o=Array.isArray(e)&&(Array.isArray(e[0])||t.isPlainObject(e[0]))?i.rows.add(e):i.row.add(e);return(r===n||r)&&i.draw(),o.flatten().toArray()},this.fnAdjustColumnSizing=function(t){var e=this.api(!0).columns.adjust(),r=e.settings()[0],i=r.oScroll;t===n||t?e.draw(!1):\"\"===i.sX&&\"\"===i.sY||Bt(r)},this.fnClearTable=function(t){var e=this.api(!0).clear();(t===n||t)&&e.draw()},this.fnClose=function(t){this.api(!0).row(t).child.hide()},this.fnDeleteRow=function(t,e,r){var i=this.api(!0),o=i.rows(t),a=o.settings()[0],l=a.aoData[o[0][0]];return o.remove(),e&&e.call(this,a,l),(r===n||r)&&i.draw(),l},this.fnDestroy=function(t){this.api(!0).destroy(t)},this.fnDraw=function(t){this.api(!0).draw(t)},this.fnFilter=function(t,e,r,i,o,a){var l=this.api(!0);null===e||e===n?l.search(t,r,i,a):l.column(e).search(t,r,i,a),l.draw()},this.fnGetData=function(t,e){var r=this.api(!0);if(t!==n){var i=t.nodeName?t.nodeName.toLowerCase():\"\";return e!==n||\"td\"==i||\"th\"==i?r.cell(t,e).data():r.row(t).data()||null}return r.data().toArray()},this.fnGetNodes=function(t){var e=this.api(!0);return t!==n?e.row(t).node():e.rows().nodes().flatten().toArray()},this.fnGetPosition=function(t){var e=this.api(!0),r=t.nodeName.toUpperCase();if(\"TR\"==r)return e.row(t).index();if(\"TD\"==r||\"TH\"==r){var n=e.cell(t).index();return[n.row,n.columnVisible,n.column]}return null},this.fnIsOpen=function(t){return this.api(!0).row(t).child.isShown()},this.fnOpen=function(t,e,r){return this.api(!0).row(t).child(e,r).show().child()[0]},this.fnPageChange=function(t,e){var r=this.api(!0).page(t);(e===n||e)&&r.draw(!1)},this.fnSetColumnVis=function(t,e,r){var i=this.api(!0).column(t).visible(e);(r===n||r)&&i.columns.adjust().draw()},this.fnSettings=function(){return ae(this[i.iApiIndex])},this.fnSort=function(t){this.api(!0).order(t).draw()},this.fnSortListener=function(t,e,r){this.api(!0).order.listener(t,e,r)},this.fnUpdate=function(t,e,r,i,o){var a=this.api(!0);return r===n||null===r?a.row(e).data(t):a.cell(e,r).data(t),(o===n||o)&&a.columns.adjust(),(i===n||i)&&a.draw(),0},this.fnVersionCheck=i.fnVersionCheck;var r=this,a=e===n,l=this.length;for(var s in a&&(e={}),this.oApi=this.internal=i.internal,h.ext.internal)s&&(this[s]=Le(s));return this.each((function(){var i,o=l>1?se({},e,!0):e,s=0,d=this.getAttribute(\"id\"),c=!1,p=h.defaults,u=t(this);if(\"table\"==this.nodeName.toLowerCase()){A(p),j(p.column),E(p,p,!0),E(p.column,p.column,!0),E(p,t.extend(o,u.data()),!0);var m=h.settings;for(s=0,i=m.length;s<i;s++){var f=m[s];if(f.nTable==this||f.nTHead&&f.nTHead.parentNode==this||f.nTFoot&&f.nTFoot.parentNode==this){var g=o.bRetrieve!==n?o.bRetrieve:p.bRetrieve,b=o.bDestroy!==n?o.bDestroy:p.bDestroy;if(a||g)return f.oInstance;if(b){f.oInstance.fnDestroy();break}return void le(f,0,\"Cannot reinitialise DataTable\",3)}if(f.sTableId==this.id){m.splice(s,1);break}}null!==d&&\"\"!==d||(d=\"DataTables_Table_\"+h.ext._unique++,this.id=d);var _=t.extend(!0,{},h.models.oSettings,{sDestroyWidth:u[0].style.width,sInstance:d,sTableId:d});_.nTable=this,_.oApi=r.internal,_.oInit=o,m.push(_),_.oInstance=1===r.length?r:u.dataTable(),A(o),D(o.oLanguage),o.aLengthMenu&&!o.iDisplayLength&&(o.iDisplayLength=Array.isArray(o.aLengthMenu[0])?o.aLengthMenu[0][0]:o.aLengthMenu[0]),o=se(t.extend(!0,{},p),o),he(_.oFeatures,o,[\"bPaginate\",\"bLengthChange\",\"bFilter\",\"bSort\",\"bSortMulti\",\"bInfo\",\"bProcessing\",\"bAutoWidth\",\"bSortClasses\",\"bServerSide\",\"bDeferRender\"]),he(_,o,[\"asStripeClasses\",\"ajax\",\"fnServerData\",\"fnFormatNumber\",\"sServerMethod\",\"aaSorting\",\"aaSortingFixed\",\"aLengthMenu\",\"sPaginationType\",\"sAjaxSource\",\"sAjaxDataPr
"/*! DataTables Bootstrap 4 integration\n",
" * ©2011-2017 SpryMedia Ltd - datatables.net/license\n",
" */n=function(t,e,r,n){\"use strict\";var i=t.fn.dataTable;return t.extend(!0,i.defaults,{dom:\"<'row'<'col-sm-12 col-md-6'l><'col-sm-12 col-md-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>\",renderer:\"bootstrap\"}),t.extend(i.ext.classes,{sWrapper:\"dataTables_wrapper dt-bootstrap4\",sFilterInput:\"form-control form-control-sm\",sLengthSelect:\"custom-select custom-select-sm form-control form-control-sm\",sProcessing:\"dataTables_processing card\",sPageButton:\"paginate_button page-item\"}),i.ext.renderer.pageButton.bootstrap=function(e,o,a,l,h,s){var d,c,p,u=new i.Api(e),m=e.oClasses,f=e.oLanguage.oPaginate,g=e.oLanguage.oAria.paginate||{},b=0,_=function(r,n){var i,o,l,p,x=function(e){e.preventDefault(),t(e.currentTarget).hasClass(\"disabled\")||u.page()==e.data.action||u.page(e.data.action).draw(\"page\")};for(i=0,o=n.length;i<o;i++)if(p=n[i],Array.isArray(p))_(r,p);else{switch(d=\"\",c=\"\",p){case\"ellipsis\":d=\"&#x2026;\",c=\"disabled\";break;case\"first\":d=f.sFirst,c=p+(h>0?\"\":\" disabled\");break;case\"previous\":d=f.sPrevious,c=p+(h>0?\"\":\" disabled\");break;case\"next\":d=f.sNext,c=p+(h<s-1?\"\":\" disabled\");break;case\"last\":d=f.sLast,c=p+(h<s-1?\"\":\" disabled\");break;default:d=p+1,c=h===p?\"active\":\"\"}d&&(l=t(\"<li>\",{class:m.sPageButton+\" \"+c,id:0===a&&\"string\"==typeof p?e.sTableId+\"_\"+p:null}).append(t(\"<a>\",{href:\"#\",\"aria-controls\":e.sTableId,\"aria-label\":g[p],\"data-dt-idx\":b,tabindex:e.iTabIndex,class:\"page-link\"}).html(d)).appendTo(r),e.oApi._fnBindAction(l,{action:p},x),b++)}};try{p=t(o).find(r.activeElement).data(\"dt-idx\")}catch(t){}_(t(o).empty().html('<ul class=\"pagination\"/>').children(\"ul\"),l),p!==n&&t(o).find(\"[data-dt-idx=\"+p+\"]\").trigger(\"focus\")},i},t.exports=function(t,e){return t||(t=window),e&&e.fn.dataTable||(e=r(22)(t,e).$),n(e,0,t.document)}},function(t,e,r){\"use strict\";r.d(e,\"a\",(function(){return s}));var n,i=r(3),o=r.n(i),a=r(0),l=r.n(a),h=(n=function(t,e){return(n=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])})(t,e)},function(t,e){function r(){this.constructor=t}n(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}),s=function(t){function e(e){var r=t.call(this,e)||this;return r.context_menu_div=l.a.createRef(),r.trigger_callbacks=[],r.onContextMenu=function(t){this.show(t.pageX,t.pageY,\"\"),t.preventDefault(),t.stopPropagation()}.bind(r),r.state={visible:!1,column:\"\",top:0,left:0},r.hide=function(){this.state.visible&&this.setState({visible:!1})}.bind(r),o()(window).on(\"click\",r.hide),r}return h(e,t),e.prototype.addCallback=function(t,e){this.trigger_callbacks.push({cb:t,obj:e})},e.prototype.removeCallbacks=function(t){this.trigger_callbacks=this.trigger_callbacks.filter((function(e){return e.obj!=t}))},e.prototype.show=function(t,e,r){var n=o()(this.context_menu_div.current.parentElement).offset();this.setState({top:Math.max(0,e-10-n.top),left:Math.max(0,t-90-n.left),visible:!0,column:r})},e.prototype.componentWillUnmount=function(){o()(window).off(\"click\",this.hide)},e.prototype.componentDidUpdate=function(t,e){var r=this.context_menu_div.current;if(r.style.display=this.state.visible?\"block\":\"none\",r.style.top=this.state.top+\"px\",r.style.left=this.state.left+\"px\",r.classList.toggle(\"show\",this.state.visible),this.state.visible&&!e.visible||this.state.column!=e.column){r.innerHTML=\"\";var n=this;this.trigger_callbacks.forEach((function(t){t.cb(n.state.column,r)}))}},e.prototype.render=function(){return l.a.createElement(\"div\",{ref:this.context_menu_div,className:\"dropdown-menu dropdown-menu-sm context-menu\",style:{fontSize:16}})},e}(l.a.Component)},function(t,e,r){\"use strict\";r.d(e,\"a\",(function(){return S}));var n,i=r(3),o=r.n(i),a=r(0),l=r.n(a),h=r(1),s=r(6),d=r(5),c=r.n(d),p=r(8),u=r(11),m=r(2),f=(n=function(t,e){return(n=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e
"/*! ColReorder 1.5.3\n",
" * ©2010-2020 SpryMedia Ltd - datatables.net/license\n",
" */n=function(t,e,r,n){\"use strict\";var i=t.fn.dataTable;function o(t){for(var e=[],r=0,n=t.length;r<n;r++)e[t[r]]=r;return e}function a(t,e,r){var n=t.splice(e,1)[0];t.splice(r,0,n)}function l(t,e,r){for(var n=[],i=0,o=t.childNodes.length;i<o;i++)1==t.childNodes[i].nodeType&&n.push(t.childNodes[i]);var a=n[e];null!==r?t.insertBefore(a,n[r]):t.appendChild(a)}t.fn.dataTableExt.oApi.fnColReorder=function(e,r,i,h,s){var d,c,p,u,m,f,g,b=e.aoColumns.length,_=function(t,e,r){if(t[e]&&\"function\"!=typeof t[e]){var n=t[e].split(\".\"),i=n.shift();isNaN(1*i)||(t[e]=r[1*i]+\".\"+n.join(\".\"))}};if(r!=i)if(r<0||r>=b)this.oApi._fnLog(e,1,\"ColReorder 'from' index is out of bounds: \"+r);else if(i<0||i>=b)this.oApi._fnLog(e,1,\"ColReorder 'to' index is out of bounds: \"+i);else{var x=[];for(d=0,c=b;d<c;d++)x[d]=d;a(x,r,i);var v=o(x);for(d=0,c=e.aaSorting.length;d<c;d++)e.aaSorting[d][0]=v[e.aaSorting[d][0]];if(null!==e.aaSortingFixed)for(d=0,c=e.aaSortingFixed.length;d<c;d++)e.aaSortingFixed[d][0]=v[e.aaSortingFixed[d][0]];for(d=0,c=b;d<c;d++){for(p=0,u=(g=e.aoColumns[d]).aDataSort.length;p<u;p++)g.aDataSort[p]=v[g.aDataSort[p]];g.idx=v[g.idx]}for(t.each(e.aLastSort,(function(t,r){e.aLastSort[t].src=v[r.src]})),d=0,c=b;d<c;d++)\"number\"==typeof(g=e.aoColumns[d]).mData?g.mData=v[g.mData]:t.isPlainObject(g.mData)&&(_(g.mData,\"_\",v),_(g.mData,\"filter\",v),_(g.mData,\"sort\",v),_(g.mData,\"type\",v));if(e.aoColumns[r].bVisible){var k=this.oApi._fnColumnIndexToVisible(e,r),y=null;for(d=i<r?i:i+1;null===y&&d<b;)y=this.oApi._fnColumnIndexToVisible(e,d),d++;for(d=0,c=(f=e.nTHead.getElementsByTagName(\"tr\")).length;d<c;d++)l(f[d],k,y);if(null!==e.nTFoot)for(d=0,c=(f=e.nTFoot.getElementsByTagName(\"tr\")).length;d<c;d++)l(f[d],k,y);for(d=0,c=e.aoData.length;d<c;d++)null!==e.aoData[d].nTr&&l(e.aoData[d].nTr,k,y)}for(a(e.aoColumns,r,i),d=0,c=b;d<c;d++)e.oApi._fnColumnOptions(e,d,{});for(a(e.aoPreSearchCols,r,i),d=0,c=e.aoData.length;d<c;d++){var w=e.aoData[d],S=w.anCells;if(S)for(a(S,r,i),p=0,m=S.length;p<m;p++)S[p]&&S[p]._DT_CellIndex&&(S[p]._DT_CellIndex.column=p);\"dom\"!==w.src&&Array.isArray(w._aData)&&a(w._aData,r,i)}for(d=0,c=e.aoHeader.length;d<c;d++)a(e.aoHeader[d],r,i);if(null!==e.aoFooter)for(d=0,c=e.aoFooter.length;d<c;d++)a(e.aoFooter[d],r,i);for((s||s===n)&&t.fn.dataTable.Api(e).rows().invalidate(),d=0,c=b;d<c;d++)t(e.aoColumns[d].nTh).off(\".DT\"),this.oApi._fnSortAttachListener(e,e.aoColumns[d].nTh,d);t(e.oInstance).trigger(\"column-reorder.dt\",[e,{from:r,to:i,mapping:v,drop:h,iFrom:r,iTo:i,aiInvertMapping:v}])}};var h=function(e,r){var n=new t.fn.dataTable.Api(e).settings()[0];if(n._colReorder)return n._colReorder;!0===r&&(r={});var i=t.fn.dataTable.camelToHungarian;return i&&(i(h.defaults,h.defaults,!0),i(h.defaults,r||{})),this.s={dt:null,enable:null,init:t.extend(!0,{},h.defaults,r),fixed:0,fixedRight:0,reorderCallback:null,mouse:{startX:-1,startY:-1,offsetX:-1,offsetY:-1,target:-1,targetIndex:-1,fromIndex:-1},aoTargets:[]},this.dom={drag:null,pointer:null},this.s.enable=this.s.init.bEnable,this.s.dt=n,this.s.dt._colReorder=this,this._fnConstruct(),this};return t.extend(h.prototype,{fnEnable:function(t){if(!1===t)return fnDisable();this.s.enable=!0},fnDisable:function(){this.s.enable=!1},fnReset:function(){return this._fnOrderColumns(this.fnOrder()),this},fnGetCurrentOrder:function(){return this.fnOrder()},fnOrder:function(e,r){var i,a,l=[],h=this.s.dt.aoColumns;if(e===n){for(i=0,a=h.length;i<a;i++)l.push(h[i]._ColReorder_iOrigCol);return l}if(r){var s=this.fnOrder();for(i=0,a=e.length;i<a;i++)l.push(t.inArray(e[i],s));e=l}return this._fnOrderColumns(o(e)),this},fnTranspose:function(e,r){r||(r=\"toCurrent\");var n=this.fnOrder(),i=this.s.dt.aoColumns;return\"toCurrent\"===r?Array.isArray(e)?t.map(e,(function(e){return t.inArray(e,n)})):t.inArray(e,n):Array.isArray(e)?t.map(e,(function(t){return i[t]._ColReorder_iOrigCol})):i[e]._ColReorder_iOrigCol},_fnConstruct:function(){var e,r=this,n=this.s.dt.aoColumns.length,i=this.s.dt.nTable;for(this.s.init.iFixedColumns&&(this.s.fixed=this.s.init.iFixedColumns),thi
"/*! Buttons for DataTables 1.6.5\n",
" * ©2016-2020 SpryMedia Ltd - datatables.net/license\n",
" */n=function(t,e,r,n){\"use strict\";var i=t.fn.dataTable,o=0,a=0,l=i.ext.buttons;function h(e,r,n){t.fn.animate?e.stop().fadeIn(r,n):(e.css(\"display\",\"block\"),n&&n.call(e))}function s(e,r,n){t.fn.animate?e.stop().fadeOut(r,n):(e.css(\"display\",\"none\"),n&&n.call(e))}var d,c=function(e,r){if(!(this instanceof c))return function(t){return new c(t,e).container()};void 0===r&&(r={}),!0===r&&(r={}),Array.isArray(r)&&(r={buttons:r}),this.c=t.extend(!0,{},c.defaults,r),r.buttons&&(this.c.buttons=r.buttons),this.s={dt:new i.Api(e),buttons:[],listenKeys:\"\",namespace:\"dtb\"+o++},this.dom={container:t(\"<\"+this.c.dom.container.tag+\"/>\").addClass(this.c.dom.container.className)},this._constructor()};t.extend(c.prototype,{action:function(t,e){var r=this._nodeToButton(t);return e===n?r.conf.action:(r.conf.action=e,this)},active:function(e,r){var i=this._nodeToButton(e),o=this.c.dom.button.active,a=t(i.node);return r===n?a.hasClass(o):(a.toggleClass(o,r===n||r),this)},add:function(t,e){var r=this.s.buttons;if(\"string\"==typeof e){for(var i=e.split(\"-\"),o=this.s,a=0,l=i.length-1;a<l;a++)o=o.buttons[1*i[a]];r=o.buttons,e=1*i[i.length-1]}return this._expandButton(r,t,o!==n,e),this._draw(),this},container:function(){return this.dom.container},disable:function(e){var r=this._nodeToButton(e);return t(r.node).addClass(this.c.dom.button.disabled).attr(\"disabled\",!0),this},destroy:function(){t(\"body\").off(\"keyup.\"+this.s.namespace);var e,r,n=this.s.buttons.slice();for(e=0,r=n.length;e<r;e++)this.remove(n[e].node);this.dom.container.remove();var i=this.s.dt.settings()[0];for(e=0,r=i.length;e<r;e++)if(i.inst===this){i.splice(e,1);break}return this},enable:function(e,r){if(!1===r)return this.disable(e);var n=this._nodeToButton(e);return t(n.node).removeClass(this.c.dom.button.disabled).removeAttr(\"disabled\"),this},name:function(){return this.c.name},node:function(e){if(!e)return this.dom.container;var r=this._nodeToButton(e);return t(r.node)},processing:function(e,r){var i=this.s.dt,o=this._nodeToButton(e);return r===n?t(o.node).hasClass(\"processing\"):(t(o.node).toggleClass(\"processing\",r),t(i.table().node()).triggerHandler(\"buttons-processing.dt\",[r,i.button(e),i,t(e),o.conf]),this)},remove:function(e){var r=this._nodeToButton(e),n=this._nodeToHost(e),i=this.s.dt;if(r.buttons.length)for(var o=r.buttons.length-1;o>=0;o--)this.remove(r.buttons[o].node);r.conf.destroy&&r.conf.destroy.call(i.button(e),i,t(e),r.conf),this._removeKey(r.conf),t(r.node).remove();var a=t.inArray(r,n);return n.splice(a,1),this},text:function(e,r){var i=this._nodeToButton(e),o=this.c.dom.collection.buttonLiner,a=i.inCollection&&o&&o.tag?o.tag:this.c.dom.buttonLiner.tag,l=this.s.dt,h=t(i.node),s=function(t){return\"function\"==typeof t?t(l,h,i.conf):t};return r===n?s(i.conf.text):(i.conf.text=r,a?h.children(a).html(s(r)):h.html(s(r)),this)},_constructor:function(){var e=this,n=this.s.dt,i=n.settings()[0],o=this.c.buttons;i._buttons||(i._buttons=[]),i._buttons.push({inst:this,name:this.c.name});for(var a=0,l=o.length;a<l;a++)this.add(o[a]);n.on(\"destroy\",(function(t,r){r===i&&e.destroy()})),t(\"body\").on(\"keyup.\"+this.s.namespace,(function(t){if(!r.activeElement||r.activeElement===r.body){var n=String.fromCharCode(t.keyCode).toLowerCase();-1!==e.s.listenKeys.toLowerCase().indexOf(n)&&e._keypress(n,t)}}))},_addKey:function(e){e.key&&(this.s.listenKeys+=t.isPlainObject(e.key)?e.key.key:e.key)},_draw:function(t,e){t||(t=this.dom.container,e=this.s.buttons),t.children().detach();for(var r=0,n=e.length;r<n;r++)t.append(e[r].inserter),t.append(\" \"),e[r].buttons&&e[r].buttons.length&&this._draw(e[r].collection,e[r].buttons)},_expandButton:function(e,r,i,o){for(var a=this.s.dt,l=Array.isArray(r)?r:[r],h=0,s=l.length;h<s;h++){var d=this._resolveExtends(l[h]);if(d)if(Array.isArray(d))this._expandButton(e,d,i,o);else{var c=this._buildButton(d,i);c&&(o!==n&&null!==o?(e.splice(o,0,c),o++):e.push(c),c.conf.buttons&&(c.collection=t(\"<\"+this.c.dom.collection.tag+\"/>\"),c.conf._collection=c.collection,this._expandButton(c.buttons,c.conf.button
"/*\n",
"object-assign\n",
"(c) Sindre Sorhus\n",
"@license MIT\n",
"*/var n=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,o=Object.prototype.propertyIsEnumerable;function a(t){if(null==t)throw new TypeError(\"Object.assign cannot be called with null or undefined\");return Object(t)}t.exports=function(){try{if(!Object.assign)return!1;var t=new String(\"abc\");if(t[5]=\"de\",\"5\"===Object.getOwnPropertyNames(t)[0])return!1;for(var e={},r=0;r<10;r++)e[\"_\"+String.fromCharCode(r)]=r;if(\"0123456789\"!==Object.getOwnPropertyNames(e).map((function(t){return e[t]})).join(\"\"))return!1;var n={};return\"abcdefghijklmnopqrst\".split(\"\").forEach((function(t){n[t]=t})),\"abcdefghijklmnopqrst\"===Object.keys(Object.assign({},n)).join(\"\")}catch(t){return!1}}()?Object.assign:function(t,e){for(var r,l,h=a(t),s=1;s<arguments.length;s++){for(var d in r=Object(arguments[s]))i.call(r,d)&&(h[d]=r[d]);if(n){l=n(r);for(var c=0;c<l.length;c++)o.call(r,l[c])&&(h[l[c]]=r[l[c]])}}return h}},function(t,e,r){var n;\n",
"/*! Bootstrap 4 styling wrapper for ColReorder\n",
" * ©2018 SpryMedia Ltd - datatables.net/license\n",
" */n=function(t,e,r,n){return t.fn.dataTable},t.exports=function(t,e){return t||(t=window),e&&e.fn.dataTable||(e=r(25)(t,e).$),e.fn.dataTable.ColReorder||r(29)(t,e),n(e,0,t.document)}},function(t,e,r){var n;\n",
"/*! Bootstrap integration for DataTables' Buttons\n",
" * ©2016 SpryMedia Ltd - datatables.net/license\n",
" */n=function(t,e,r,n){\"use strict\";var i=t.fn.dataTable;return t.extend(!0,i.Buttons.defaults,{dom:{container:{className:\"dt-buttons btn-group flex-wrap\"},button:{className:\"btn btn-secondary\"},collection:{tag:\"div\",className:\"dropdown-menu\",button:{tag:\"a\",className:\"dt-button dropdown-item\",active:\"active\",disabled:\"disabled\"}}},buttonCreated:function(e,r){return e.buttons?t('<div class=\"btn-group\"/>').append(r):r}}),i.ext.buttons.collection.className+=\" dropdown-toggle\",i.ext.buttons.collection.rightAlignClassName=\"dropdown-menu-right\",i.Buttons},t.exports=function(t,e){return t||(t=window),e&&e.fn.dataTable||(e=r(25)(t,e).$),e.fn.dataTable.Buttons||r(30)(t,e),n(e,0,t.document)}},function(t,e,r){var n=r(50),i=r(51),o=r(52),a=r(53),l=r(54),h=r(55),s=r(56);s.alea=n,s.xor128=i,s.xorwow=o,s.xorshift7=a,s.xor4096=l,s.tychei=h,t.exports=s},function(module,__webpack_exports__,__webpack_require__){\"use strict\";__webpack_require__.d(__webpack_exports__,\"a\",(function(){return WebserverDataProvider}));var jquery__WEBPACK_IMPORTED_MODULE_0__=__webpack_require__(3),jquery__WEBPACK_IMPORTED_MODULE_0___default=__webpack_require__.n(jquery__WEBPACK_IMPORTED_MODULE_0__),_types__WEBPACK_IMPORTED_MODULE_1__=__webpack_require__(2),react__WEBPACK_IMPORTED_MODULE_2__=__webpack_require__(0),react__WEBPACK_IMPORTED_MODULE_2___default=__webpack_require__.n(react__WEBPACK_IMPORTED_MODULE_2__),_hiplot_scss__WEBPACK_IMPORTED_MODULE_3__=__webpack_require__(5),_hiplot_scss__WEBPACK_IMPORTED_MODULE_3___default=__webpack_require__.n(_hiplot_scss__WEBPACK_IMPORTED_MODULE_3__),__extends=(extendStatics=function(t,e){return(extendStatics=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(t,e){t.__proto__=e}||function(t,e){for(var r in e)e.hasOwnProperty(r)&&(t[r]=e[r])})(t,e)},function(t,e){function r(){this.constructor=t}extendStatics(t,e),t.prototype=null===e?Object.create(e):(r.prototype=e.prototype,new r)}),extendStatics,PSTATE_LOAD_URI=\"load_uri\",RunsSelectionTextArea=function(t){function e(e){var r=t.call(this,e)||this;return r.textarea=react__WEBPACK_IMPORTED_MODULE_2___default.a.createRef(),r.state={value:e.initialValue},r}return __extends(e,t),e.prototype.onInput=function(){var t=this.textarea.current;if(this.props.hasFocus||!this.props.minimizeWhenOutOfFocus)return t.style.height=\"auto\",void(t.style.height=t.scrollHeight+\"px\");t.style.height=\"55px\"},e.prototype.onKeyDown=function(t){13!==t.which||t.shiftKey||(this.props.onSubmit(this.textarea.current.value),this.props.onFocusChange(!1),t.preventDefault())},e.prototype.onFocusChange=function(t){\"focus\"==t.type?this.props.onFocusChange(!0):\"blur\"==t.type&&this.props.onFocusChange(!1)},e.prototype.componentDidMount=function(){this.onInput()},e.prototype.componentDidUpdate=function(){this.onInput()},e.prototype.render=function(){var t=this;return react__WEBPACK_IMPORTED_MODULE_2___default.a.createElement(\"textarea\",{style:{height:\"55px\",flex:1,minWidth:\"100px\"},ref:this.textarea,className:_hiplot_scss__WEBPACK_IMPORTED_MODULE_3___default.a.runsSelectionTextarea,disabled:!this.props.enabled,value:this.state.value,onKeyDown:this.onKeyDown.bind(this),onInput:this.onInput.bind(this),onChange:function(e){return t.setState({value:e.target.value})},onFocus:this.onFocusChange.bind(this),onBlur:this.onFocusChange.bind(this),placeholder:\"Experiments to load\"})},e}(react__WEBPACK_IMPORTED_MODULE_2___default.a.Component);function loadURIFromWebServer(uri){return new Promise((function(resolve,reject){jquery__WEBPACK_IMPORTED_MODULE_0___default.a.get(\"/data?uri=\"+encodeURIComponent(uri),resolve,\"json\").fail((function(data){if(4==data.readyState&&200==data.status)console.log(\"Unable to parse JSON with JS default decoder (Maybe it contains NaNs?). Using eval\"),resolve(eval(\"(\"+data.responseText+\")\"));else{if(0==data.status)return void resolve({error:\"Network error\"});reject(data)}}))}))}var WebserverDataProvider=function(t){function e(e){var r=t.call(this,e)||this;return r.state={uri:r.props.persistentState.get(PSTATE_LOAD_URI)},r}
"/** @license React v17.0.2\n",
" * react-dom.production.min.js\n",
" *\n",
" * Copyright (c) Facebook, Inc. and its affiliates.\n",
" *\n",
" * This source code is licensed under the MIT license found in the\n",
" * LICENSE file in the root directory of this source tree.\n",
" */var n=r(0),i=r(34),o=r(44);function a(t){for(var e=\"https://reactjs.org/docs/error-decoder.html?invariant=\"+t,r=1;r<arguments.length;r++)e+=\"&args[]=\"+encodeURIComponent(arguments[r]);return\"Minified React error #\"+t+\"; visit \"+e+\" for the full message or use the non-minified dev environment for full errors and additional helpful warnings.\"}if(!n)throw Error(a(227));var l=new Set,h={};function s(t,e){d(t,e),d(t+\"Capture\",e)}function d(t,e){for(h[t]=e,t=0;t<e.length;t++)l.add(e[t])}var c=!(\"undefined\"==typeof window||void 0===window.document||void 0===window.document.createElement),p=/^[:A-Z_a-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD][:A-Z_a-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD\\-.0-9\\u00B7\\u0300-\\u036F\\u203F-\\u2040]*$/,u=Object.prototype.hasOwnProperty,m={},f={};function g(t,e,r,n,i,o,a){this.acceptsBooleans=2===e||3===e||4===e,this.attributeName=n,this.attributeNamespace=i,this.mustUseProperty=r,this.propertyName=t,this.type=e,this.sanitizeURL=o,this.removeEmptyString=a}var b={};\"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style\".split(\" \").forEach((function(t){b[t]=new g(t,0,!1,t,null,!1,!1)})),[[\"acceptCharset\",\"accept-charset\"],[\"className\",\"class\"],[\"htmlFor\",\"for\"],[\"httpEquiv\",\"http-equiv\"]].forEach((function(t){var e=t[0];b[e]=new g(e,1,!1,t[1],null,!1,!1)})),[\"contentEditable\",\"draggable\",\"spellCheck\",\"value\"].forEach((function(t){b[t]=new g(t,2,!1,t.toLowerCase(),null,!1,!1)})),[\"autoReverse\",\"externalResourcesRequired\",\"focusable\",\"preserveAlpha\"].forEach((function(t){b[t]=new g(t,2,!1,t,null,!1,!1)})),\"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture disableRemotePlayback formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope\".split(\" \").forEach((function(t){b[t]=new g(t,3,!1,t.toLowerCase(),null,!1,!1)})),[\"checked\",\"multiple\",\"muted\",\"selected\"].forEach((function(t){b[t]=new g(t,3,!0,t,null,!1,!1)})),[\"capture\",\"download\"].forEach((function(t){b[t]=new g(t,4,!1,t,null,!1,!1)})),[\"cols\",\"rows\",\"size\",\"span\"].forEach((function(t){b[t]=new g(t,6,!1,t,null,!1,!1)})),[\"rowSpan\",\"start\"].forEach((function(t){b[t]=new g(t,5,!1,t.toLowerCase(),null,!1,!1)}));var _=/[\\-:]([a-z])/g;function x(t){return t[1].toUpperCase()}function v(t,e,r,n){var i=b.hasOwnProperty(e)?b[e]:null;(null!==i?0===i.type:!n&&(2<e.length&&(\"o\"===e[0]||\"O\"===e[0])&&(\"n\"===e[1]||\"N\"===e[1])))||(function(t,e,r,n){if(null==e||function(t,e,r,n){if(null!==r&&0===r.type)return!1;switch(typeof e){case\"function\":case\"symbol\":return!0;case\"boolean\":return!n&&(null!==r?!r.acceptsBooleans:\"data-\"!==(t=t.toLowerCase().slice(0,5))&&\"aria-\"!==t);default:return!1}}(t,e,r,n))return!0;if(n)return!1;if(null!==r)switch(r.type){case 3:return!e;case 4:return!1===e;case 5:return isNaN(e);case 6:return isNaN(e)||1>e}return!1}(e,r,i,n)&&(r=null),n||null===i?function(t){return!!u.call(f,t)||!u.call(m,t)&&(p.test(t)?f[t]=!0:(m[t]=!0,!1))}(e)&&(null===r?t.removeAttribute(e):t.setAttribute(e,\"\"+r)):i.mustUseProperty?t[i.propertyName]=null===r?3!==i.type&&\"\":r:(e=i.attributeName,n=i.attributeNamespace,null===r?t.removeAttribute(e):(r=3===(i=i.type)||4===i&&!0===r?\"\":\"\"+r,n?t.setAttributeNS(n,e,r):t.setAttribute(e,r))))}\"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientatio
"/** @license React v17.0.2\n",
" * react.production.min.js\n",
" *\n",
" * Copyright (c) Facebook, Inc. and its affiliates.\n",
" *\n",
" * This source code is licensed under the MIT license found in the\n",
" * LICENSE file in the root directory of this source tree.\n",
" */var n=r(34),i=60103,o=60106;e.Fragment=60107,e.StrictMode=60108,e.Profiler=60114;var a=60109,l=60110,h=60112;e.Suspense=60113;var s=60115,d=60116;if(\"function\"==typeof Symbol&&Symbol.for){var c=Symbol.for;i=c(\"react.element\"),o=c(\"react.portal\"),e.Fragment=c(\"react.fragment\"),e.StrictMode=c(\"react.strict_mode\"),e.Profiler=c(\"react.profiler\"),a=c(\"react.provider\"),l=c(\"react.context\"),h=c(\"react.forward_ref\"),e.Suspense=c(\"react.suspense\"),s=c(\"react.memo\"),d=c(\"react.lazy\")}var p=\"function\"==typeof Symbol&&Symbol.iterator;function u(t){for(var e=\"https://reactjs.org/docs/error-decoder.html?invariant=\"+t,r=1;r<arguments.length;r++)e+=\"&args[]=\"+encodeURIComponent(arguments[r]);return\"Minified React error #\"+t+\"; visit \"+e+\" for the full message or use the non-minified dev environment for full errors and additional helpful warnings.\"}var m={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},f={};function g(t,e,r){this.props=t,this.context=e,this.refs=f,this.updater=r||m}function b(){}function _(t,e,r){this.props=t,this.context=e,this.refs=f,this.updater=r||m}g.prototype.isReactComponent={},g.prototype.setState=function(t,e){if(\"object\"!=typeof t&&\"function\"!=typeof t&&null!=t)throw Error(u(85));this.updater.enqueueSetState(this,t,e,\"setState\")},g.prototype.forceUpdate=function(t){this.updater.enqueueForceUpdate(this,t,\"forceUpdate\")},b.prototype=g.prototype;var x=_.prototype=new b;x.constructor=_,n(x,g.prototype),x.isPureReactComponent=!0;var v={current:null},k=Object.prototype.hasOwnProperty,y={key:!0,ref:!0,__self:!0,__source:!0};function w(t,e,r){var n,o={},a=null,l=null;if(null!=e)for(n in void 0!==e.ref&&(l=e.ref),void 0!==e.key&&(a=\"\"+e.key),e)k.call(e,n)&&!y.hasOwnProperty(n)&&(o[n]=e[n]);var h=arguments.length-2;if(1===h)o.children=r;else if(1<h){for(var s=Array(h),d=0;d<h;d++)s[d]=arguments[d+2];o.children=s}if(t&&t.defaultProps)for(n in h=t.defaultProps)void 0===o[n]&&(o[n]=h[n]);return{$$typeof:i,type:t,key:a,ref:l,props:o,_owner:v.current}}function S(t){return\"object\"==typeof t&&null!==t&&t.$$typeof===i}var M=/\\/+/g;function C(t,e){return\"object\"==typeof t&&null!==t&&null!=t.key?function(t){var e={\"=\":\"=0\",\":\":\"=2\"};return\"$\"+t.replace(/[=:]/g,(function(t){return e[t]}))}(\"\"+t.key):e.toString(36)}function T(t,e,r,n,a){var l=typeof t;\"undefined\"!==l&&\"boolean\"!==l||(t=null);var h=!1;if(null===t)h=!0;else switch(l){case\"string\":case\"number\":h=!0;break;case\"object\":switch(t.$$typeof){case i:case o:h=!0}}if(h)return a=a(h=t),t=\"\"===n?\".\"+C(h,0):n,Array.isArray(a)?(r=\"\",null!=t&&(r=t.replace(M,\"$&/\")+\"/\"),T(a,e,r,\"\",(function(t){return t}))):null!=a&&(S(a)&&(a=function(t,e){return{$$typeof:i,type:t.type,key:e,ref:t.ref,props:t.props,_owner:t._owner}}(a,r+(!a.key||h&&h.key===a.key?\"\":(\"\"+a.key).replace(M,\"$&/\")+\"/\")+t)),e.push(a)),1;if(h=0,n=\"\"===n?\".\":n+\":\",Array.isArray(t))for(var s=0;s<t.length;s++){var d=n+C(l=t[s],s);h+=T(l,e,r,d,a)}else if(\"function\"==typeof(d=function(t){return null===t||\"object\"!=typeof t?null:\"function\"==typeof(t=p&&t[p]||t[\"@@iterator\"])?t:null}(t)))for(t=d.call(t),s=0;!(l=t.next()).done;)h+=T(l=l.value,e,r,d=n+C(l,s++),a);else if(\"object\"===l)throw e=\"\"+t,Error(u(31,\"[object Object]\"===e?\"object with keys {\"+Object.keys(t).join(\", \")+\"}\":e));return h}function E(t,e,r){if(null==t)return t;var n=[],i=0;return T(t,n,\"\",\"\",(function(t){return e.call(r,t,i++)})),n}function D(t){if(-1===t._status){var e=t._result;e=e(),t._status=0,t._result=e,e.then((function(e){0===t._status&&(e=e.default,t._status=1,t._result=e)}),(function(e){0===t._status&&(t._status=2,t._result=e)}))}if(1===t._status)return t._result;throw t._result}var N={current:null};function A(){var t=N.current;if(null===t)throw Error(u(321));return t}var j={ReactCurrentDispatcher:N,ReactCurrentBatchConfig:{transition:0},ReactCurrentOwner:v,IsSomeRendererActing:{current:!1},assign:n};e.Children={map:E,forEach:function(t,e,r){E(t,(
"/** @license React v0.20.2\n",
" * scheduler.production.min.js\n",
" *\n",
" * Copyright (c) Facebook, Inc. and its affiliates.\n",
" *\n",
" * This source code is licensed under the MIT license found in the\n",
" * LICENSE file in the root directory of this source tree.\n",
" */var n,i,o,a;if(\"object\"==typeof performance&&\"function\"==typeof performance.now){var l=performance;e.unstable_now=function(){return l.now()}}else{var h=Date,s=h.now();e.unstable_now=function(){return h.now()-s}}if(\"undefined\"==typeof window||\"function\"!=typeof MessageChannel){var d=null,c=null,p=function(){if(null!==d)try{var t=e.unstable_now();d(!0,t),d=null}catch(t){throw setTimeout(p,0),t}};n=function(t){null!==d?setTimeout(n,0,t):(d=t,setTimeout(p,0))},i=function(t,e){c=setTimeout(t,e)},o=function(){clearTimeout(c)},e.unstable_shouldYield=function(){return!1},a=e.unstable_forceFrameRate=function(){}}else{var u=window.setTimeout,m=window.clearTimeout;if(\"undefined\"!=typeof console){var f=window.cancelAnimationFrame;\"function\"!=typeof window.requestAnimationFrame&&console.error(\"This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills\"),\"function\"!=typeof f&&console.error(\"This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills\")}var g=!1,b=null,_=-1,x=5,v=0;e.unstable_shouldYield=function(){return e.unstable_now()>=v},a=function(){},e.unstable_forceFrameRate=function(t){0>t||125<t?console.error(\"forceFrameRate takes a positive int between 0 and 125, forcing frame rates higher than 125 fps is not supported\"):x=0<t?Math.floor(1e3/t):5};var k=new MessageChannel,y=k.port2;k.port1.onmessage=function(){if(null!==b){var t=e.unstable_now();v=t+x;try{b(!0,t)?y.postMessage(null):(g=!1,b=null)}catch(t){throw y.postMessage(null),t}}else g=!1},n=function(t){b=t,g||(g=!0,y.postMessage(null))},i=function(t,r){_=u((function(){t(e.unstable_now())}),r)},o=function(){m(_),_=-1}}function w(t,e){var r=t.length;t.push(e);t:for(;;){var n=r-1>>>1,i=t[n];if(!(void 0!==i&&0<C(i,e)))break t;t[n]=e,t[r]=i,r=n}}function S(t){return void 0===(t=t[0])?null:t}function M(t){var e=t[0];if(void 0!==e){var r=t.pop();if(r!==e){t[0]=r;t:for(var n=0,i=t.length;n<i;){var o=2*(n+1)-1,a=t[o],l=o+1,h=t[l];if(void 0!==a&&0>C(a,r))void 0!==h&&0>C(h,a)?(t[n]=h,t[l]=r,n=l):(t[n]=a,t[o]=r,n=o);else{if(!(void 0!==h&&0>C(h,r)))break t;t[n]=h,t[l]=r,n=l}}}return e}return null}function C(t,e){var r=t.sortIndex-e.sortIndex;return 0!==r?r:t.id-e.id}var T=[],E=[],D=1,N=null,A=3,j=!1,z=!1,L=!1;function I(t){for(var e=S(E);null!==e;){if(null===e.callback)M(E);else{if(!(e.startTime<=t))break;M(E),e.sortIndex=e.expirationTime,w(T,e)}e=S(E)}}function O(t){if(L=!1,I(t),!z)if(null!==S(T))z=!0,n(P);else{var e=S(E);null!==e&&i(O,e.startTime-t)}}function P(t,r){z=!1,L&&(L=!1,o()),j=!0;var n=A;try{for(I(r),N=S(T);null!==N&&(!(N.expirationTime>r)||t&&!e.unstable_shouldYield());){var a=N.callback;if(\"function\"==typeof a){N.callback=null,A=N.priorityLevel;var l=a(N.expirationTime<=r);r=e.unstable_now(),\"function\"==typeof l?N.callback=l:N===S(T)&&M(T),I(r)}else M(T);N=S(T)}if(null!==N)var h=!0;else{var s=S(E);null!==s&&i(O,s.startTime-r),h=!1}return h}finally{N=null,A=n,j=!1}}var R=a;e.unstable_IdlePriority=5,e.unstable_ImmediatePriority=1,e.unstable_LowPriority=4,e.unstable_NormalPriority=3,e.unstable_Profiling=null,e.unstable_UserBlockingPriority=2,e.unstable_cancelCallback=function(t){t.callback=null},e.unstable_continueExecution=function(){z||j||(z=!0,n(P))},e.unstable_getCurrentPriorityLevel=function(){return A},e.unstable_getFirstCallbackNode=function(){return S(T)},e.unstable_next=function(t){switch(A){case 1:case 2:case 3:var e=3;break;default:e=A}var r=A;A=e;try{return t()}finally{A=r}},e.unstable_pauseExecution=function(){},e.unstable_requestPaint=R,e.unstable_runWithPriority=function(t,e){switch(t){case 1:case 2:case 3:case 4:case 5:break;default:t=3}var r=A;A=t;try{return e()}finally{A=r}},e.unstable_scheduleCallback=function(t,r,a){var l=e.unstable_now();switch(\"object\"==typeof a&&null!==a?a=\"number\"==typeof(a=a.delay)&&0<a?l+a:l:a=l,t){case 1:var h=-1;break;case 2:h=250;break;case 5:h=1073741823;break;case 4:h=1e4;break;default:h=5e3}return t={id:D++,callba
"//# sourceMappingURL=hiplot.bundle.js.map\n",
"// Local variables can't be accessed in other cells, so let's\n",
"// manually create a global variable\n",
"Object.assign(window, {'hiplot': hiplot});\n"
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<!DOCTYPE html>\n",
"<html>\n",
" <head>\n",
" <meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\"/>\n",
" <title>HiPlot</title>\n",
" <link rel=\"icon\" href=\"static/icon.png\" />\n",
" </head>\n",
" <body style=\"margin:0px\">\n",
" <div id=\"hiplot_537012f4494b4fc9a61290e8f0deae57\" style=\"background-color: white\"><div style=\"text-align: center\">Loading HiPlot...</div>\n",
" <noscript>\n",
" HiPlot needs JavaScript to run\n",
" </noscript>\n",
" </div>\n",
"\n",
" <script type=\"text/javascript\" ></script>\n",
" <script type=\"text/javascript\">\n",
" function _hiplot_setup() {\n",
" if (typeof hiplot == \"undefined\") {\n",
" console.info(\"waiting for HiPlot bundle to be loaded...\");\n",
" setTimeout(_hiplot_setup, 1000);\n",
" return;\n",
" }\n",
" var options = {};\n",
" /*ON_LOAD_SCRIPT_INJECT*/\n",
"const comm_id = \"comm_cd7cf0\";\n",
"try {\n",
" console.log(\"Setting up communication channel with Jupyter: \", comm_id);\n",
" const comm = Jupyter.notebook.kernel.comm_manager.new_comm(comm_id, {'type': 'hello'});\n",
" var comm_message_id = 0;\n",
" function send_data_change(type, data) {\n",
" comm.send({\n",
" 'type': type,\n",
" 'message_id': comm_message_id,\n",
" 'data': data,\n",
" });\n",
" comm_message_id += 1;\n",
" };\n",
" Object.assign(options, {\"on_change\": {\n",
" \"selected_uids\": send_data_change,\n",
" \"brush_extents\": send_data_change,\n",
" }});\n",
"}\n",
"catch(err) {\n",
" console.warn('Unable to create Javascript <-> Python communication channel' +\n",
" ' (are you in a Jupyter notebook? Jupyter labs is *not* supported!)');\n",
"}\n",
" \n",
" Object.assign(options, eval('(' + \"{\\\"dataProviderName\\\": \\\"none\\\", \\\"experiment\\\": {\\\"parameters_definition\\\": {}, \\\"colormap\\\": \\\"interpolateTurbo\\\", \\\"colorby\\\": null, \\\"display_data\\\": {}, \\\"datapoints\\\": [{\\\"uid\\\": \\\"avg-acc\\\", \\\"values\\\": {\\\"0\\\": \\\"nor_conv_3x3\\\", \\\"1\\\": \\\"nor_conv_3x3\\\", \\\"2\\\": \\\"nor_conv_3x3\\\", \\\"3\\\": \\\"nor_conv_3x3\\\", \\\"4\\\": \\\"nor_conv_3x3\\\", \\\"5\\\": \\\"nor_conv_3x3\\\", \\\"acc\\\": 93.76}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"disc-acc\\\", \\\"values\\\": {\\\"0\\\": \\\"none\\\", \\\"1\\\": \\\"none\\\", \\\"2\\\": \\\"none\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"none\\\", \\\"5\\\": \\\"none\\\", \\\"acc\\\": 86.45333333333333}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"darts-pt\\\", \\\"values\\\": {\\\"0\\\": \\\"skip_connect\\\", \\\"1\\\": \\\"skip_connect\\\", \\\"2\\\": \\\"nor_conv_3x3\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"skip_connect\\\", \\\"5\\\": \\\"skip_connect\\\", \\\"acc\\\": 80.57}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"zc-pt\\\", \\\"values\\\": {\\\"0\\\": \\\"nor_conv_3x3\\\", \\\"1\\\": \\\"nor_conv_3x3\\\", \\\"2\\\": \\\"nor_conv_1x1\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"nor_conv_3x3\\\", \\\"5\\\": \\\"nor_conv_1x1\\\", \\\"acc\\\": 94.195}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"disc-zc\\\", \\\"values\\\": {\\\"0\\\": \\\"skip_connect\\\", \\\"1\\\": \\\"skip_connect\\\", \\\"2\\\": \\\"nor_conv_1x1\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"skip_connect\\\", \\\"5\\\": \\\"skip_connect\\\", \\\"acc\\\": 77.04333333333334}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"best-acc\\\", \\\"values\\\": {\\\"0\\\": \\\"nor_conv_3x3\\\", \\\"1\\\": \\\"nor_conv_3x3\\\", \\\"2\\\": \\\"nor_conv_3x3\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"nor_conv_3x3\\\", \\\"5\\\": \\\"nor_conv_1x1\\\", \\\"acc\\\": 94.37333333333333}, \\\"from_uid\\\": null}, {\\\"uid\\\": \\\"darts\\\", \\\"values\\\": {\\\"0\\\": \\\"skip_connect\\\", \\\"1\\\": \\\"skip_connect\\\", \\\"2\\\": \\\"skip_connect\\\", \\\"3\\\": \\\"skip_connect\\\", \\\"4\\\": \\\"skip_connect\\\", \\\"5\\\": \\\"skip_connect\\\", \\\"acc\\\": 54.29666666666667}, \\\"from_uid\\\": null}]}, \\\"persistentState\\\": null}\" + ')'));\n",
" \n",
" var hiplot_instance = hiplot.render(document.getElementById(\"hiplot_537012f4494b4fc9a61290e8f0deae57\"), options);\n",
" Object.assign(window, {'hiplot_last_instance': hiplot_instance}); // For debugging\n",
" /*AFTER_SETUP_SCRIPT_INJECT*/\n",
" }\n",
" if (document.readyState == \"complete\") {\n",
" _hiplot_setup();\n",
" }\n",
" else {\n",
" document.addEventListener(\"DOMContentLoaded\", _hiplot_setup);\n",
" }\n",
" </script>\n",
" </body>\n",
"</html>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"<hiplot.ipython.IPythonExperimentDisplayed at 0x7f1475058320>"
]
},
"execution_count": 478,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import hiplot as hip\n",
"dps = []\n",
"for uid,data in plot_data:\n",
" dp = hip.Datapoint(uid=uid, values=data)\n",
" dps.append(dp)\n",
"hip.Experiment(datapoints=dps).display(force_full_width=False)\n"
]
},
{
"cell_type": "code",
"execution_count": 479,
"id": "98e2dab7",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"avg-acc rank = 166 / 15625 top 1.1 % error= 6.24\n",
"disc-acc rank = 12744 / 15625 top 81.6 % error= 13.55\n",
"darts-pt rank = 13770 / 15625 top 88.1 % error= 19.43\n",
"zc-pt rank = 14 / 15625 top 0.1 % error= 5.81\n",
"disc-zc rank = 14274 / 15625 top 91.4 % error= 22.96\n",
"best-acc rank = 1 / 15625 top 0.0 % error= 5.63\n",
"darts rank = 15231 / 15625 top 97.5 % error= 45.7\n"
]
}
],
"source": [
"acc_d = pickle.load(open('nb2_test_acc_cf10.p','rb'))\n",
"for m in metrics:\n",
" best_ops = best_archs[m]\n",
" selected_acc = stat.mean(acc_d[tuple(best_ops)]) # avg accuracy\n",
" rank = 1\n",
" for k,v in acc_d.items():\n",
" if stat.mean(v) > selected_acc:\n",
" rank += 1\n",
" print(m,'rank = ',rank,'/',len(acc_d), 'top', round(rank/len(acc_d)*100,1), '% error=', round(100-selected_acc,2))"
]
},
{
"cell_type": "code",
"execution_count": 482,
"id": "ad23ee51",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"avg-acc rank = 92 / 15625 top 0.6 % error= 6.02\n",
"disc-acc rank = 12803 / 15625 top 81.9 % error= 13.43\n",
"darts-pt rank = 13754 / 15625 top 88.0 % error= 18.75\n",
"zc-pt rank = 16 / 15625 top 0.1 % error= 5.72\n",
"disc-zc rank = 14273 / 15625 top 91.3 % error= 22.39\n",
"best-acc rank = 1 / 15625 top 0.0 % error= 5.32\n",
"darts rank = 15141 / 15625 top 96.9 % error= 37.45\n"
]
}
],
"source": [
"acc_d = pickle.load(open('nb2_test_acc_cf10.p','rb'))\n",
"for m in metrics:\n",
" best_ops = best_archs[m]\n",
" selected_acc = max(acc_d[tuple(best_ops)]) # best acc\n",
" rank = 1\n",
" for k,v in acc_d.items():\n",
" if max(v) > selected_acc:\n",
" rank += 1\n",
" print(m,'rank = ',rank,'/',len(acc_d), 'top', round(rank/len(acc_d)*100,1), '% error=', round(100-selected_acc,2))"
]
},
{
"cell_type": "code",
"execution_count": 483,
"id": "660d4d81",
"metadata": {},
"outputs": [],
"source": [
"def get_top(k):\n",
" return f\"\"\"\\\\begin{{table}}[t]\n",
"\\\\centering\n",
"\\\\caption {{Raw values of operation scoring functions to reproduce Figure~\\\\ref{{metric_correlation}}.}}\n",
"\\\\footnotesize \n",
"\\\\begin{{threeparttable}} \n",
"\\\\begin{{tabular}}{{@{{}}lcccccc@{{}}}} \n",
"\\\\toprule\n",
"& $edge\\\\backslash op$ & none & skip\\_connect & nor\\_conv\\_1x1 & nor\\_conv\\_3x3 & avg\\_pool\\_3x3 \\\\\\\\ \\\\toprule\"\"\"\n",
"\n",
"def get_bot(k):\n",
" return f\"\"\"\\\\bottomrule\n",
"\\\\end{{tabular}}\n",
"\\\\end{{threeparttable}}\n",
"\\\\label{{tbl:raw-{k}}}\n",
"\\\\end{{table}}\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 487,
"id": "db405e8d",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\\begin{table}[t]\n",
"\\centering\n",
"\\caption {Raw values of operation scoring functions to reproduce Figure~\\ref{metric_correlation}.}\n",
"\\footnotesize \n",
"\\begin{threeparttable} \n",
"\\begin{tabular}{@{}lcccccc@{}} \n",
"\\toprule\n",
"& $edge\\backslash op$ & none & skip\\_connect & nor\\_conv\\_1x1 & nor\\_conv\\_3x3 & avg\\_pool\\_3x3 \\\\ \\toprule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{avg-acc}}} & 0 & 77.36 & 81.02 & 83.81 & \\secondbest{86.38} & \\best{87.32} \\\\\n",
" & 1 & 80.03 & 83.11 & \\secondbest{85.23} & \\best{85.99} & 81.52 \\\\\n",
" & 2 & 82.9 & 82.44 & \\secondbest{84.05} & \\best{84.49} & 81.98 \\\\\n",
" & 3 & 74.02 & 85.17 & \\secondbest{87.3} & \\best{88.28} & 81.38 \\\\\n",
" & 4 & 80.14 & 83.05 & \\secondbest{85.09} & \\best{85.7} & 81.89 \\\\\n",
" & 5 & 77.61 & 83.43 & \\secondbest{86.18} & \\best{86.95} & 81.74 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{disc-acc}}} & 0 & \\best{83.27} & \\secondbest{82.24} & 65.0 & 71.76 & 54.31 \\\\\n",
" & 1 & \\best{84.94} & 83.23 & 73.23 & 76.77 & \\secondbest{83.45} \\\\\n",
" & 2 & \\best{83.87} & \\secondbest{83.73} & 77.33 & 76.83 & 83.25 \\\\\n",
" & 3 & 65.77 & \\best{84.44} & 75.82 & \\secondbest{78.68} & 62.7 \\\\\n",
" & 4 & \\best{83.57} & 82.03 & 75.02 & 76.09 & \\secondbest{82.56} \\\\\n",
" & 5 & \\best{83.95} & \\secondbest{82.45} & 66.69 & 71.36 & 80.31 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{darts-pt}}} & 0 & -85.43 & \\best{-17.02} & -78.13 & \\secondbest{-59.09} & -85.34 \\\\\n",
" & 1 & -85.52 & \\best{-36.1} & -84.39 & \\secondbest{-80.95} & -85.49 \\\\\n",
" & 2 & -85.51 & \\secondbest{-80.29} & -81.86 & \\best{-77.68} & -85.32 \\\\\n",
" & 3 & -85.49 & \\best{-9.86} & -81.79 & \\secondbest{-59.18} & -85.48 \\\\\n",
" & 4 & -85.45 & \\best{-51.15} & -78.84 & \\secondbest{-64.64} & -85.14 \\\\\n",
" & 5 & -85.54 & \\best{-32.43} & -81.04 & \\secondbest{-72.75} & -85.51 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{zc-pt}}} & 0 & -3455.23 & -3449.9 & \\secondbest{-3449.54} & \\best{-3441.82} & -3461.18 \\\\\n",
" & 1 & -3452.15 & -3448.7 & \\secondbest{-3441.81} & \\best{-3440.65} & -3453.74 \\\\\n",
" & 2 & -3446.52 & -3447.61 & \\best{-3435.46} & \\secondbest{-3436.4} & -3449.28 \\\\\n",
" & 3 & -3453.81 & \\best{-3435.99} & \\secondbest{-3444.04} & -3445.6 & -3447.07 \\\\\n",
" & 4 & -3451.06 & -3449.8 & \\secondbest{-3442.63} & \\best{-3441.13} & -3453.31 \\\\\n",
" & 5 & -3450.97 & -3448.21 & \\best{-3440.8} & \\secondbest{-3443.24} & -3452.99 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{disc-zc}}} & 0 & 3331.01 & \\best{3445.49} & 3366.88 & \\secondbest{3437.55} & 3423.18 \\\\\n",
" & 1 & 3429.07 & \\best{3435.75} & 3407.87 & \\secondbest{3434.58} & 3421.44 \\\\\n",
" & 2 & 3428.8 & 3423.36 & \\best{3440.93} & \\secondbest{3437.29} & 3416.89 \\\\\n",
" & 3 & 3408.99 & \\best{3464.05} & 3359.89 & 3382.18 & \\secondbest{3431.81} \\\\\n",
" & 4 & \\secondbest{3433.99} & \\best{3435.57} & 3424.47 & 3431.14 & 3423.15 \\\\\n",
" & 5 & \\secondbest{3434.42} & \\best{3437.66} & 3418.57 & 3397.52 & 3424.17 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{best-acc}}} & 0 & 94.15 & 94.18 & \\secondbest{94.44} & \\best{94.68} & 93.86 \\\\\n",
" & 1 & 94.24 & 94.16 & \\secondbest{94.49} & \\best{94.68} & 94.09 \\\\\n",
" & 2 & 94.25 & 94.43 & \\secondbest{94.49} & \\best{94.68} & 94.19 \\\\\n",
" & 3 & \\secondbest{94.16} & \\best{94.68} & 94.03 & 94.04 & 93.85 \\\\\n",
" & 4 & 94.29 & 94.18 & \\secondbest{94.56} & \\best{94.68} & 94.23 \\\\\n",
" & 5 & 94.05 & 94.16 & \\best{94.68} & \\secondbest{94.56} & 94.1 \\\\\n",
"\\midrule\n",
"\\parbox[t]{2mm}{\\multirow{6}{*}{\\rotatebox[origin=c]{90}{darts}}} & 0 & 0.14 & \\best{0.48} & 0.13 & \\secondbest{0.18} & 0.07 \\\\\n",
" & 1 & 0.12 & \\best{0.55} & 0.11 & \\secondbest{0.12} & 0.09 \\\\\n",
" & 2 & \\secondbest{0.24} & \\best{0.33} & 0.15 & 0.17 & 0.11 \\\\\n",
" & 3 & 0.06 & \\best{0.65} & 0.08 & \\secondbest{0.13} & 0.07 \\\\\n",
" & 4 & 0.12 & \\best{0.48} & 0.13 & \\secondbest{0.17} & 0.1 \\\\\n",
" & 5 & \\secondbest{0.16} & \\best{0.49} & 0.12 & 0.14 & 0.09 \\\\\n",
"\\midrule\n",
"\\bottomrule\n",
"\\end{tabular}\n",
"\\end{threeparttable}\n",
"\\label{tbl:raw-op-scores}\n",
"\\end{table}\n",
"\n"
]
}
],
"source": [
"import copy\n",
"print(get_top(k))\n",
"for k,v in sq_op_strengths.items():\n",
" #print(k)\n",
" for i,r in enumerate(v):\n",
" if i == 0:\n",
" print(f'\\\\parbox[t]{{2mm}}{{\\\\multirow{{6}}{{*}}{{\\\\rotatebox[origin=c]{{90}}{{{k}}}}}}} & ', end='')\n",
" print(f'{i} & ', end='')\n",
" else:\n",
" print(f' & {i} & ', end='')\n",
" for c in r:\n",
" first = max(r)\n",
" rr = copy.copy(r)\n",
" rr.sort()\n",
" second = rr[-2]\n",
" if c == first:\n",
" print(f'\\\\best{{{round(c,2)}}}', end='')\n",
" elif c == second:\n",
" print(f'\\\\secondbest{{{round(c,2)}}}', end='')\n",
" else:\n",
" print(round(c,2), end='')\n",
" if c != r[-1]:\n",
" print(' & ', end='')\n",
" else:\n",
" print(' \\\\\\\\')\n",
" print('\\\\midrule')\n",
"print(get_bot('op-scores'))\n",
"print()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fa6b0d77",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 22,
"id": "6a8f2a31",
"metadata": {},
"outputs": [],
"source": [
"from itertools import product\n",
"x =list(product([0,1,2,3], repeat=4))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "76f46a37",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3), (0, 1, 0, 0), (0, 1, 0, 1), (0, 1, 0, 2), (0, 1, 0, 3), (0, 1, 1, 0), (0, 1, 1, 1), (0, 1, 1, 2), (0, 1, 1, 3), (0, 1, 2, 0), (0, 1, 2, 1), (0, 1, 2, 2), (0, 1, 2, 3), (0, 1, 3, 0), (0, 1, 3, 1), (0, 1, 3, 2), (0, 1, 3, 3), (0, 2, 0, 0), (0, 2, 0, 1), (0, 2, 0, 2), (0, 2, 0, 3), (0, 2, 1, 0), (0, 2, 1, 1), (0, 2, 1, 2), (0, 2, 1, 3), (0, 2, 2, 0), (0, 2, 2, 1), (0, 2, 2, 2), (0, 2, 2, 3), (0, 2, 3, 0), (0, 2, 3, 1), (0, 2, 3, 2), (0, 2, 3, 3), (0, 3, 0, 0), (0, 3, 0, 1), (0, 3, 0, 2), (0, 3, 0, 3), (0, 3, 1, 0), (0, 3, 1, 1), (0, 3, 1, 2), (0, 3, 1, 3), (0, 3, 2, 0), (0, 3, 2, 1), (0, 3, 2, 2), (0, 3, 2, 3), (0, 3, 3, 0), (0, 3, 3, 1), (0, 3, 3, 2), (0, 3, 3, 3), (1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3), (1, 1, 0, 0), (1, 1, 0, 1), (1, 1, 0, 2), (1, 1, 0, 3), (1, 1, 1, 0), (1, 1, 1, 1), (1, 1, 1, 2), (1, 1, 1, 3), (1, 1, 2, 0), (1, 1, 2, 1), (1, 1, 2, 2), (1, 1, 2, 3), (1, 1, 3, 0), (1, 1, 3, 1), (1, 1, 3, 2), (1, 1, 3, 3), (1, 2, 0, 0), (1, 2, 0, 1), (1, 2, 0, 2), (1, 2, 0, 3), (1, 2, 1, 0), (1, 2, 1, 1), (1, 2, 1, 2), (1, 2, 1, 3), (1, 2, 2, 0), (1, 2, 2, 1), (1, 2, 2, 2), (1, 2, 2, 3), (1, 2, 3, 0), (1, 2, 3, 1), (1, 2, 3, 2), (1, 2, 3, 3), (1, 3, 0, 0), (1, 3, 0, 1), (1, 3, 0, 2), (1, 3, 0, 3), (1, 3, 1, 0), (1, 3, 1, 1), (1, 3, 1, 2), (1, 3, 1, 3), (1, 3, 2, 0), (1, 3, 2, 1), (1, 3, 2, 2), (1, 3, 2, 3), (1, 3, 3, 0), (1, 3, 3, 1), (1, 3, 3, 2), (1, 3, 3, 3), (2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3), (2, 1, 0, 0), (2, 1, 0, 1), (2, 1, 0, 2), (2, 1, 0, 3), (2, 1, 1, 0), (2, 1, 1, 1), (2, 1, 1, 2), (2, 1, 1, 3), (2, 1, 2, 0), (2, 1, 2, 1), (2, 1, 2, 2), (2, 1, 2, 3), (2, 1, 3, 0), (2, 1, 3, 1), (2, 1, 3, 2), (2, 1, 3, 3), (2, 2, 0, 0), (2, 2, 0, 1), (2, 2, 0, 2), (2, 2, 0, 3), (2, 2, 1, 0), (2, 2, 1, 1), (2, 2, 1, 2), (2, 2, 1, 3), (2, 2, 2, 0), (2, 2, 2, 1), (2, 2, 2, 2), (2, 2, 2, 3), (2, 2, 3, 0), (2, 2, 3, 1), (2, 2, 3, 2), (2, 2, 3, 3), (2, 3, 0, 0), (2, 3, 0, 1), (2, 3, 0, 2), (2, 3, 0, 3), (2, 3, 1, 0), (2, 3, 1, 1), (2, 3, 1, 2), (2, 3, 1, 3), (2, 3, 2, 0), (2, 3, 2, 1), (2, 3, 2, 2), (2, 3, 2, 3), (2, 3, 3, 0), (2, 3, 3, 1), (2, 3, 3, 2), (2, 3, 3, 3), (3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3), (3, 1, 0, 0), (3, 1, 0, 1), (3, 1, 0, 2), (3, 1, 0, 3), (3, 1, 1, 0), (3, 1, 1, 1), (3, 1, 1, 2), (3, 1, 1, 3), (3, 1, 2, 0), (3, 1, 2, 1), (3, 1, 2, 2), (3, 1, 2, 3), (3, 1, 3, 0), (3, 1, 3, 1), (3, 1, 3, 2), (3, 1, 3, 3), (3, 2, 0, 0), (3, 2, 0, 1), (3, 2, 0, 2), (3, 2, 0, 3), (3, 2, 1, 0), (3, 2, 1, 1), (3, 2, 1, 2), (3, 2, 1, 3), (3, 2, 2, 0), (3, 2, 2, 1), (3, 2, 2, 2), (3, 2, 2, 3), (3, 2, 3, 0), (3, 2, 3, 1), (3, 2, 3, 2), (3, 2, 3, 3), (3, 3, 0, 0), (3, 3, 0, 1), (3, 3, 0, 2), (3, 3, 0, 3), (3, 3, 1, 0), (3, 3, 1, 1), (3, 3, 1, 2), (3, 3, 1, 3), (3, 3, 2, 0), (3, 3, 2, 1), (3, 3, 2, 2), (3, 3, 2, 3), (3, 3, 3, 0), (3, 3, 3, 1), (3, 3, 3, 2), (3, 3, 3, 3)]\n"
]
}
],
"source": [
"print(x)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e633562a",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "darts",
"language": "python",
"name": "darts"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue Copy Permalink