|
| 1 | +#!/usr/bin/env python |
| 2 | +# coding=utf-8 |
| 3 | + |
| 4 | +import os |
| 5 | +import sys |
| 6 | +import re |
| 7 | +import json |
| 8 | +import numpy |
| 9 | + |
| 10 | +FLAG = True |
| 11 | + |
| 12 | +#FLAG=False |
| 13 | + |
| 14 | + |
| 15 | +def get_all_vars(var_file): |
| 16 | + var_list = [] |
| 17 | + |
| 18 | + with open(var_file, 'r') as fin: |
| 19 | + for line in fin: |
| 20 | + line = line.strip('\n') |
| 21 | + if len(line) > 0: |
| 22 | + var_list.append(line) |
| 23 | + |
| 24 | + return var_list |
| 25 | + |
| 26 | + |
| 27 | +def get_data_from_log(online_log_file, origin_file): |
| 28 | + insid_list = [] |
| 29 | + with open(origin_file, 'r') as fin: |
| 30 | + for line in fin: |
| 31 | + insid = line.strip('\n').split('\t')[0] |
| 32 | + insid_list.append(insid) |
| 33 | + # print("insid_list:", insid_list) |
| 34 | + |
| 35 | + res = {} |
| 36 | + pattern = r'\[([^\]]*)\]' |
| 37 | + with open(online_log_file, 'r') as fin: |
| 38 | + for line in fin: |
| 39 | + insid = '' |
| 40 | + var_name = '' |
| 41 | + length = 0 |
| 42 | + data = [] |
| 43 | + ele_list = re.findall(pattern, line.strip('\n')) |
| 44 | + for ele in ele_list: |
| 45 | + if ele.find(':') < 0: |
| 46 | + continue |
| 47 | + key, value = ele.split(':') |
| 48 | + if key == 'batchIdx': |
| 49 | + insid = insid_list[int(value)] |
| 50 | + if key == 'name': |
| 51 | + var_name = value |
| 52 | + if key == 'shape': |
| 53 | + dim1, dim2 = value.split(',') |
| 54 | + length = int(dim1) * int(dim2) |
| 55 | + if key == 'data': |
| 56 | + data = [float(x) for x in value.split(',')] |
| 57 | + assert length == len(data) |
| 58 | + if insid not in res: |
| 59 | + res[insid] = {} |
| 60 | + res[insid][var_name] = (length, data) |
| 61 | + # print("online log:", res.keys()) |
| 62 | + |
| 63 | + return res |
| 64 | + |
| 65 | + |
| 66 | +def get_data_from_model(model_dump_file): |
| 67 | + res = {} |
| 68 | + |
| 69 | + with open(model_dump_file, 'r') as fin: |
| 70 | + for line in fin: |
| 71 | + line = line.strip('\n').split('\t') |
| 72 | + insid = line[0] |
| 73 | + res[insid] = {} |
| 74 | + for var in line[2:]: |
| 75 | + var_name, length, data = var.split(':', 2) |
| 76 | + length = int(length) |
| 77 | + data = [float(x) for x in data.split(':')] |
| 78 | + assert length == len(data) |
| 79 | + res[insid][var_name] = (length, data) |
| 80 | + # print("offline log:", res.keys()) |
| 81 | + return res |
| 82 | + |
| 83 | + |
| 84 | +def onoff_var_diff(log_data, model_data, var_name): |
| 85 | + res = {'labels': [], 'values': []} |
| 86 | + diff_ins_list = [] |
| 87 | + |
| 88 | + same_insid_counts = 0 |
| 89 | + all_diff = [] |
| 90 | + for insid in log_data.keys(): |
| 91 | + if insid in model_data: |
| 92 | + same_insid_counts += 1 |
| 93 | + log_val = log_data[insid][var_name] |
| 94 | + if var_name in model_data[insid]: |
| 95 | + model_val = model_data[insid][var_name] |
| 96 | + assert log_val[0] == model_val[0] |
| 97 | + for i in range(log_val[0]): |
| 98 | + all_diff.append(abs(log_val[1][i] - model_val[1][i])) |
| 99 | + print(insid, log_val[1][i], model_val[1][i]) |
| 100 | + if abs(log_val[1][i] - model_val[1][i]) > 0.01: |
| 101 | + diff_ins_list.append((insid, log_val[1], model_val[1])) |
| 102 | + print("diff_ins_list:", diff_ins_list) |
| 103 | + # print("all_diff:", all_diff) |
| 104 | + all_diff = numpy.asarray(all_diff) |
| 105 | + total = same_insid_counts |
| 106 | + print("same ins total: {}".format(same_insid_counts)) |
| 107 | + no_diff = numpy.sum(all_diff < 0.000001) |
| 108 | + if no_diff: |
| 109 | + res['labels'].append('无diff') |
| 110 | + res['values'].append(no_diff) |
| 111 | + print("无diff: {}, {}".format(no_diff, float(no_diff) / total)) |
| 112 | + labels = [ |
| 113 | + '个位diff', '十分位diff', '百分位diff', '千分位diff', '万分位diff', '十万分位diff', |
| 114 | + '百万分位diff' |
| 115 | + ] |
| 116 | + multis = [1, 10, 100, 1000, 10000, 100000, 1000000] |
| 117 | + for label, multi in zip(labels, multis): |
| 118 | + diff_counts = numpy.sum(all_diff * multi > 1) |
| 119 | + if diff_counts: |
| 120 | + res['labels'].append(label) |
| 121 | + res['values'].append(diff_counts) |
| 122 | + print("{}: {}, {}".format(label, diff_counts, |
| 123 | + float(diff_counts) / total)) |
| 124 | + # return json.dumps(res, ensure_ascii=False) if res['values'] else "" |
| 125 | + return diff_ins_list |
| 126 | + |
| 127 | + |
| 128 | +def onoff_max_diff(log_data, model_data, ins_id): |
| 129 | + log_ins = log_data[ins_id] |
| 130 | + model_ins = model_data[ins_id] |
| 131 | + for var_name in model_ins: |
| 132 | + max_diff = 0.0 |
| 133 | + if var_name not in log_ins: |
| 134 | + print( |
| 135 | + 'var {} is not in online log'.format(var_name), |
| 136 | + file=sys.stderr) |
| 137 | + continue |
| 138 | + if log_ins[var_name][0] != model_ins[var_name][0]: |
| 139 | + print( |
| 140 | + 'The length of {} is wrong, online is {}, offline is {}'. |
| 141 | + format(var_name, log_ins[var_name][0], model_ins[var_name][0]), |
| 142 | + file=sys.stderr) |
| 143 | + continue |
| 144 | + for i in range(log_ins[var_name][0]): |
| 145 | + diff = abs(log_ins[var_name][1][i] - model_ins[var_name][1][i]) |
| 146 | + if max_diff < diff: |
| 147 | + max_diff = diff |
| 148 | + if max_diff > 2e-5: |
| 149 | + print("ins_id:{}, var_name:{}: {}".format(ins_id, var_name, max_diff)) |
| 150 | + |
| 151 | + |
| 152 | +if __name__ == '__main__': |
| 153 | + import argparse |
| 154 | + parser = argparse.ArgumentParser() |
| 155 | + parser.add_argument('-l', help='线上日志文件', dest='online_log_file', required=True) |
| 156 | + parser.add_argument( |
| 157 | + '-m', help='模型dump文件', dest='model_dump_file', required=True) |
| 158 | + parser.add_argument('-v', help='所有var文件', dest='var_file', required=True) |
| 159 | + parser.add_argument('-o', help='offline原始数据文件', dest='origin_file', required=True) |
| 160 | + args = parser.parse_args() |
| 161 | + log_data = get_data_from_log(args.online_log_file, args.origin_file) |
| 162 | + model_data = get_data_from_model(args.model_dump_file) |
| 163 | + var_list = get_all_vars(args.var_file) |
| 164 | + diff_ins_list = onoff_var_diff(log_data, model_data, var_list[-1]) |
| 165 | + # diff_ins_list = [str(i) for i in range(1, 11)] |
| 166 | + for ins in diff_ins_list: |
| 167 | + onoff_max_diff(log_data, model_data, ins) |
| 168 | + # print('<pie-charts>{}</pie-charts>'.format(res)) |
0 commit comments