#!/usr/bin/env python3 # coding: utf-8 # Copyright (C) 1994-2021 Altair Engineering, Inc. # For more information, contact Altair at www.altair.com. # # This file is part of both the OpenPBS software ("OpenPBS") # and the PBS Professional ("PBS Pro") software. # # Open Source License Information: # # OpenPBS is free software. You can redistribute it and/or modify it under # the terms of the GNU Affero General Public License as published by the # Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # OpenPBS is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public # License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # # Commercial License Information: # # PBS Pro is commercially licensed software that shares a common core with # the OpenPBS software. For a copy of the commercial license terms and # conditions, go to: (http://www.pbspro.com/agreement.html) or contact the # Altair Legal Department. # # Altair's dual-license business model allows companies, individuals, and # organizations to create proprietary derivative works of OpenPBS and # distribute them - whether embedded or bundled with other software - # under a commercial license agreement. # # Use of Altair's trademarks, including but not limited to "PBS™", # "OpenPBS®", "PBS Professional®", and "PBS Pro™" and Altair's logos is # subject to Altair's trademark licensing policies. import os import csv import json import sys import getopt import time def usage(): msg = [] msg += ['Usage: ' + os.path.basename(sys.argv[0])] msg += [' [benchmark_json_file] [tocompare_json_file] [OPTION]\n\n'] msg += [' Performance test results comparision tool'] msg += [' to generate csv and html report\n\n'] msg += ['--html-report : option to generate html report\n'] msg += ['--output-file : path to generate csv and html file\n'] msg += ['--help or -h : To display usage information\n'] msg += ['--append : Append results to an existing file\n'] print(''.join(msg)) def generate_html_report(filepath, append): """ Generate html performance comparision report """ HTML = '''
Performance tests benchmark comparision results
user: %s
host: %s
%s
''' HTML_add = ''' %s ''' if not filepath.endswith('.html'): filepath = filepath + '.html' if append: fd = open(filepath, "r") d = fd.read() fd.close() m = d.split("\n") s = "\n".join(m[:-4]) fd = open(filepath, "w+") for i in range(len(s)): fd.write(s[i]) fd.close() with open(filepath, 'a+') as fp: fp.write(HTML_add % (''.join(_data))) else: with open(filepath, 'w+') as fp: fp.write(HTML % (oldv['user'], list(oldv['machine_info'].keys())[0], _h + ''.join(_data))) def generate_csv_report(filepath, append): """ compare 2 json results and generate csv report """ if not filepath.endswith('.csv'): filepath = filepath + '.csv' if append: with open(filepath, 'a+') as fp: csv.writer(fp).writerows(mdata) else: with open(filepath, 'w+') as fp: csv.writer(fp).writerows([header] + mdata) def percent_change(nv, ov, unit): """ swap the values to find approriate percent change for units """ if unit == 'jobs/sec': a = ov ov = nv nv = a diff = ov - nv pchange = 0 if nv == 0: nv = 1 if diff > 0: pchange = (diff / nv) * 100 elif diff < 0: diff = nv - ov pchange = -(diff / nv) * 100 pchange = round(pchange, 2) return str(pchange) + '%' if __name__ == '__main__': if len(sys.argv) < 3: usage() sys.exit(1) html_report = False try: opts, args = getopt.getopt(sys.argv[3:], "h", ["help", "html-report", "output-file=", "append"]) except getopt.GetoptError as err: print(err) usage() sys.exit(1) filepath = None append = 0 for o, val in opts: if o == '--html-report': html_report = True elif o in ("-h", "--help"): usage() sys.exit(0) elif o == "--output-file": filepath = val elif o == "--append": append = 1 with open(sys.argv[1]) as fp: oldv = json.load(fp) newfiles = sys.argv[2].split(',') header = ['TestCase', 'Test Measure', 'Unit', oldv['product_version'] + ' baseline PBS'] TR = ' \n%s \n' TH = ' %s\n' TD = ' %s\n' TDR = ' %s\n' filenum = 0 mult_data = {} for newfile in newfiles: with open(newfile) as fp: newv = json.load(fp) header.extend([newv['product_version'], '% Improvement']) for k, v in sorted(oldv['avg_measurements']['testsuites'].items()): assert k in newv['avg_measurements']['testsuites'], k _ntcs = newv['avg_measurements']['testsuites'][k]['testcases'] _otcs = v['testcases'] mn = 0 for _k, _v in sorted(v['testcases'].items()): _tcn = k + '.' + _k assert _k in _ntcs, _tcn _om = _v _om = [x for x in _om if 'test_measure' in x] _om = sorted(_om, key=lambda x: x['test_measure']) _nm = _ntcs[_k] _nm = [x for x in _nm if 'test_measure' in x] _nm = sorted(_nm, key=lambda x: x['test_measure']) _nm_ms = [x['test_measure'] for x in _nm] for key, val in sorted(oldv['testsuites'].items()): assert key in newv['testsuites'], key for tc, doc in sorted(val['testcases'].items()): if _k == tc: _docs = doc['docstring'] for i, _m in enumerate(_om): data = [] _mn = _m['test_measure'] _msg = 'test measure %s missing' % _mn _msg += ' in new %s' % _tcn assert _mn in _nm_ms, _msg _os = _m['test_data']['std_dev'] _o = _m['test_data']['mean'] _omi = _m['test_data']['minimum'] _oma = _m['test_data']['maximum'] _omt = _m['test_data']['total_samples'] _oms = _m['test_data']['samples_considered'] _n = _nm[i]['test_data']['mean'] _ns = _nm[i]['test_data']['std_dev'] _nsmi = _nm[i]['test_data']['minimum'] _nsma = _nm[i]['test_data']['maximum'] _nst = _nm[i]['test_data']['total_samples'] _nss = _nm[i]['test_data']['samples_considered'] _old_vals = ('mean:' + str(round(_o, 2)) + ', std_dev:' + str(round(_os, 2)) + ', minimum:' + str(round(_omi, 2)) + ', maximum:' + str(round(_oma, 2)) + ', mean_samples:' + str(round(_omt, 2)) + ', samples_considered:' + str(round(_oms, 2))) _new_vals = ('mean:' + str(round(_n, 2)) + ', std_dev:' + str(round(_ns, 2)) + ', minimum:' + str(round(_nsmi, 2)) + ', maximum:' + str(round(_nsma, 2)) + ', mean_samples:' + str(round(_nst, 2)) + ', samples_considered:' + str(round(_nss, 2))) _row = [_tcn, _mn, _m['unit'], _old_vals] _rowadd = [_new_vals, percent_change(_n, _o, _m['unit'])] if filenum == 0: data = _row mult_data[mn] = data data.extend(_rowadd) else: data.extend(_rowadd) mult_data[mn].extend(data) mn = mn + 1 filenum += 1 mdata = [] for ind, dat in mult_data.items(): mdata.append(dat) _h = TR % ''.join([TH % x for x in header]) _data = [] _rsns = {} _adf = [] for i, d in enumerate(mdata): if d[1] in _rsns: _rsns[d[1]] += 1 else: _rsns.setdefault(d[1], 1) for i, d in enumerate(mdata): if _rsns[d[1]] > 1: if d[1] in _adf: _data.append(TR % ''.join([TD % x for x in d[2:]])) else: _d = [TDR % (_rsns[d[1]], x) for x in d[:2]] _d1 = [TD % x for x in d[2:]] _data.append(TR % ''.join(_d + _d1)) _adf.append(d[1]) else: _data.append(TR % ''.join([TD % x for x in d])) if not filepath: filepath = 'performance_test_report' if html_report: generate_html_report(filepath, append) generate_csv_report(filepath, append)