diff --git a/graph_dit/test_perf.py b/graph_dit/test_perf.py new file mode 100644 index 0000000..43c0160 --- /dev/null +++ b/graph_dit/test_perf.py @@ -0,0 +1,136 @@ +from nas_201_api import NASBench201API as API +import re +import pandas as pd +import json +import numpy as np +import argparse + +api = API('./NAS-Bench-201-v1_1-096897.pth') + +parser = argparse.ArgumentParser(description='Process some integers.') + +parser.add_argument('--file_path', type=str, default='211035.txt',) +args = parser.parse_args() + +def process_graph_data(text): + # Split the input text into sections for each graph + graph_sections = text.strip().split('nodes:') + + # Prepare lists to store data + nodes_list = [] + edges_list = [] + results_list = [] + + for section in graph_sections[1:]: + # Extract nodes + nodes_section = section.split('edges:')[0] + nodes_match = re.search(r'(tensor\(\d+\) ?)+', section) + if nodes_match: + nodes = re.findall(r'tensor\((\d+)\)', nodes_match.group(0)) + nodes_list.append(nodes) + + # Extract edges + edge_section = section.split('edges:')[1] + edges_match = re.search(r'edges:', section) + if edges_match: + edges = re.findall(r'tensor\((\d+)\)', edge_section) + edges_list.append(edges) + + # Extract the last floating point number as a result + + # Create a DataFrame to store the extracted data + data = { + 'nodes': nodes_list, + 'edges': edges_list, + } + data['nodes'] = [[int(x) for x in node] for node in data['nodes']] + data['edges'] = [[int(x) for x in edge] for edge in data['edges']] + def split_list(input_list, chunk_size): + return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)] + data['edges'] = [split_list(edge, 8) for edge in data['edges']] + + print(data) + df = pd.DataFrame(data) + print('df') + print(df['nodes'][0], df['edges'][0]) + return df + +def is_valid_nasbench201(adj, ops): + print(ops) + if ops[0] != 0 or ops[-1] != 6: + return False + for i in range(2, len(ops) - 1): + if ops[i] not in [1, 2, 3, 4, 5]: + return False + adj_mat = [ [0, 1, 1, 0, 1, 0, 0, 0], + [0, 0, 0, 1, 0, 1 ,0 ,0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 0]] + + for i in range(len(adj)): + for j in range(len(adj[i])): + if adj[i][j] not in [0, 1]: + return False + if j > i: + if adj[i][j] != adj_mat[i][j]: + return False + return True + +num_to_op = ['input', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3', 'skip_connect', 'none', 'output'] +def nodes_to_arch_str(nodes): + nodes_str = [num_to_op[node] for node in nodes] + arch_str = '|' + nodes_str[1] + '~0|+' + \ + '|' + nodes_str[2] + '~0|' + nodes_str[3] + '~1|+' +\ + '|' + nodes_str[4] + '~0|' + nodes_str[5] + '~1|' + nodes_str[6] + '~2|' + return arch_str + +filename = args.file_path + +with open('./output_graphs/' + filename, 'r') as f: + texts = f.read() + df = process_graph_data(texts) + valid = 0 + not_valid = 0 + scores = [] + dist = {'<90':0, '<91':0, '<92':0, '<93':0, '<94':0, '>94':0} + for i in range(len(df)): + nodes = df['nodes'][i] + edges = df['edges'][i] + result = is_valid_nasbench201(edges, nodes) + if result: + valid += 1 + arch_str = nodes_to_arch_str(nodes) + index = api.query_index_by_arch(arch_str) + # results = api.query_by_index(index, 'cifar10', hp='200') + # print(results) + # result = results[888].get_eval('ori-test') + res = api.get_more_info(index, 'cifar10', None, hp=200, is_random=False) + acc = res['test-accuracy'] + scores.append((index, acc)) + if acc < 90: + dist['<90'] += 1 + elif acc < 91 and acc >= 90: + dist['<91'] += 1 + elif acc < 92 and acc >= 91: + dist['<92'] += 1 + elif acc < 93 and acc >= 92: + dist['<93'] += 1 + elif acc < 94 and acc >= 93: + dist['<94'] += 1 + else: + dist['>94'] += 1 + else: + not_valid += 1 + with open('./output_graphs/' + filename + '.json', 'w') as f: + json.dump(scores, f) + print(scores) + print(valid, not_valid) + print(dist) + print("mean: ", np.mean([x[1] for x in scores])) + print("max: ", np.max([x[1] for x in scores])) + print("min: ", np.min([x[1] for x in scores])) +