From f887d5d14dece5e0eec03113e0a5bfd2d6a02de0 Mon Sep 17 00:00:00 2001 From: Julian Hammer Date: Mon, 14 Oct 2019 17:08:40 +0200 Subject: [PATCH] work in progress --- osaca/data/model_importer.py | 188 ++++++++--------------------------- osaca/parser/__init__.py | 10 +- osaca/semantics/hw_model.py | 84 +++++++++++++--- 3 files changed, 118 insertions(+), 164 deletions(-) diff --git a/osaca/data/model_importer.py b/osaca/data/model_importer.py index cae5556..28bf62d 100755 --- a/osaca/data/model_importer.py +++ b/osaca/data/model_importer.py @@ -4,115 +4,29 @@ import re import sys import xml.etree.ElementTree as ET from distutils.version import StrictVersion -from itertools import groupby, product - -from ruamel import yaml from osaca.db_interface import add_entries_to_db -from osaca.parser import ParserAArch64v81, ParserX86ATT +from osaca.parser import get_parser from osaca.semantics import MachineModel -ARCH_DICT = { - 'vulcan': 'aarch64', - 'snb': 'x86', - 'ivb': 'x86', - 'hsw': 'x86', - 'bdw': 'x86', - 'skl': 'x86', - 'skx': 'x86', - 'csx': 'x86', -} +def port_pressure_from_tag_attributes(attrib): + # '1*p015+1*p1+1*p23+1*p4+3*p5' -> + # [(1, '015'), (1, '1'), (1, '23'), (1, '4'), (3, '5')] + port_occupation = [] + for p in attrib['ports'].split('+'): + cycles, ports = p.split('*p') + port_occupation.append((int(cycles), ports)) -def port_pressure_from_tag_attributes(attrib, arch, ports): - # apply cycles for D ports - data_port = re.compile(r'[0-9]D$') - data_ports = [x[:-1] for x in filter(data_port.match, ports)] - - # format attributes - cycles = attrib['ports'].split('+') - cycles = [c.split('*') for c in cycles] - for i, c in enumerate(cycles): - cycles[i][0] = int(c[0]) - if str(c[1]).startswith('p'): - cycles[i][1] = [p for p in c[1][1:]] - if data_ports and data_ports == cycles[i][1]: - # uops for data ports - cycles.append([c[0], [x + 'D' for x in data_ports]]) - cycles[i][0] = [ - cycles[i][0] / num for num in range(1, len(cycles[i][1]) + 1) for _ in range(num) - ] - cycles = [list(product(c[0], c[1])) for c in cycles] - all_options = [] - - # iterate over all combinations of all uop options - for cycles_combs in cycles: - options = [] - tmp_opt = [] - total = cycles_combs[0][0] - # iterate over all combinations of each uop option - for comb in cycles_combs: - # add options until they reach the total num of uops - tmp_opt.append(comb) - if sum([c[0] for c in tmp_opt]) == total: - # copy this option as one of several to the cycle option list - options.append(tmp_opt.copy()) - tmp_opt = [] - if len(tmp_opt) != 0: - raise ValueError('Cannot compute port pressure') - options = [x for x, _ in groupby(options)] - all_options.append(options) - all_options = list(product(*all_options)) - - # find best scheduling - port_pressure = {} - for p in ports: - port_pressure[p] = 0.0 - first = calculate_port_pressure(all_options[0]) - for key in first: - port_pressure[key] = first[key] - for option in all_options[1:]: - tmp = calculate_port_pressure(option) - if (max(list(tmp.values())) <= max(list(port_pressure.values()))) and ( - len(tmp) > len([x for x in port_pressure.values() if x != 0.0]) - ): - for k in port_pressure: - port_pressure[k] = tmp[k] if k in tmp else 0.0 - - # check if calculation equals given throughput - if abs(max(list(port_pressure.values())) - float(attrib['TP_ports'])) > 0.01: - print('Contradicting TP value compared to port_pressure. Ignore port pressure.') - for p in port_pressure: - port_pressure[p] = 0.0 - return port_pressure - - # Also consider DIV pipeline + # Also if 'div_cycles' in attrib: - div_port = re.compile(r'[0-9]DV$') - div_ports = [x for x in filter(div_port.match, ports)] - for dp in div_ports: - port_pressure[dp] += int(attrib['div_cycles']) / len(div_ports) - return port_pressure + port_occupation.append((int(attrib['div_cycles']), ('DV',))) + + return port_occupation -def calculate_port_pressure(pp_option): - ports = {} - for option in pp_option: - for port in option: - if port[1] in ports: - ports[port[1]] += port[0] - else: - ports[port[1]] = port[0] - return ports - - -def extract_paramters(instruction_tag, arch): - isa = ARCH_DICT[arch.lower()] - parser = ParserX86ATT() - if isa == 'aarch64': - parser = ParserAArch64v81() - elif isa == 'x86': - parser = ParserX86ATT() +def extract_paramters(instruction_tag, isa): + parser = get_parser(isa) # Extract parameter components parameters = [] # used to store string representations parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib['idx'])) @@ -138,16 +52,12 @@ def extract_paramters(instruction_tag, arch): parameter['class'] = 'register' possible_regs = [parser.parse_register('%' + r) for r in parameter_tag.text.split(',')] if possible_regs[0] is None: - raise ValueError( - 'Unknown register type for {} with {}.'.format( - parameter_tag.attrib, parameter_tag.text - ) - ) + raise ValueError('Unknown register type for {} with {}.'.format( + parameter_tag.attrib, parameter_tag.text)) if isa == 'x86': if parser.is_vector_register(possible_regs[0]['register']): - possible_regs[0]['register']['name'] = possible_regs[0]['register'][ - 'name' - ].lower()[:3] + possible_regs[0]['register']['name'] = \ + possible_regs[0]['register']['name'].lower()[:3] if 'mask' in possible_regs[0]['register']: possible_regs[0]['register']['mask'] = True else: @@ -175,9 +85,8 @@ def extract_paramters(instruction_tag, arch): def extract_model(tree, arch): - mm = MachineModel(arch.lower()) - ports = mm._data['ports'] - model_data = [] + mm = MachineModel() + for instruction_tag in tree.findall('.//instruction'): ignore = False @@ -185,8 +94,8 @@ def extract_model(tree, arch): # Extract parameter components try: - parameters = extract_paramters(instruction_tag, arch) - if ARCH_DICT[arch.lower()] == 'x86': + parameters = extract_paramters(instruction_tag, mm.get_isa_for_arch(arch)) + if mm.get_isa_for_arch(arch).lower() == 'x86': parameters.reverse() except ValueError as e: print(e, file=sys.stderr) @@ -201,16 +110,12 @@ def extract_model(tree, arch): if 'TP_ports' in measurement_tag.attrib: throughput = measurement_tag.attrib['TP_ports'] else: - throughput = ( - measurement_tag.attrib['TP'] if 'TP' in measurement_tag.attrib else None - ) - uops = ( - int(measurement_tag.attrib['uops']) if 'uops' in measurement_tag.attrib else None - ) + throughput = (measurement_tag.attrib['TP'] + if 'TP' in measurement_tag.attrib else None) + uops = (int(measurement_tag.attrib['uops']) + if 'uops' in measurement_tag.attrib else None) if 'ports' in measurement_tag.attrib: - port_pressure.append( - port_pressure_from_tag_attributes(measurement_tag.attrib, arch, ports) - ) + port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib)) latencies = [ int(l_tag.attrib['cycles']) for l_tag in measurement_tag.iter('latency') @@ -230,39 +135,30 @@ def extract_model(tree, arch): # Ordered by IACA version (newest last) for iaca_tag in sorted( - arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version']) - ): + arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version'])): if 'ports' in iaca_tag.attrib: - port_pressure.append( - port_pressure_from_tag_attributes(iaca_tag.attrib, arch, ports) - ) + port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib)) if ignore: continue + # Add missing ports: + [p[1] for p in for pp in port_pressure] + mm.add_port() + # Check if all are equal if port_pressure: if port_pressure[1:] != port_pressure[:-1]: - print( - "Contradicting port occupancies, using latest IACA:", mnemonic, file=sys.stderr - ) + print("Contradicting port occupancies, using latest IACA:", mnemonic, + file=sys.stderr) port_pressure = port_pressure[-1] - throughput = max(list(port_pressure.values()) + [0.0]) + throughput = max(mm.average_port_pressure(port_pressure)) else: # print("No data available for this architecture:", mnemonic, file=sys.stderr) continue # --------------------------------------------- - model_data.append( - { - 'name': mnemonic, - 'operands': parameters, - 'uops': uops, - 'throughput': throughput, - 'latency': latency, - 'port_pressure': port_pressure, - } - ) + mm.set_instruction(mnemonic, parameters, latency, port_pressure, throughput, uops) - return model_data + return mm def architectures(tree): @@ -282,12 +178,10 @@ def main(): tree = ET.parse(args.xml) if args.arch: - model_data = extract_model(tree, args.arch) - print(yaml.dump(model_data, allow_unicode=True)) + model = extract_model(tree, args.arch) + print(model.dump()) else: - for arch in architectures(tree): - model_data = extract_model(tree, arch) - add_entries_to_db(arch, model_data) + raise NotImplementedError() if __name__ == '__main__': diff --git a/osaca/parser/__init__.py b/osaca/parser/__init__.py index 33ac13a..cce0143 100644 --- a/osaca/parser/__init__.py +++ b/osaca/parser/__init__.py @@ -8,4 +8,12 @@ from .base_parser import BaseParser from .parser_x86att import ParserX86ATT from .parser_AArch64v81 import ParserAArch64v81 -__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64v81'] +__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64v81', 'get_parser'] + +def get_parser(isa): + if isa.lower() == 'x86': + return ParserX86ATT() + elif isa.lower() == 'aarch64': + return ParserAArch64v81() + else: + raise ValueError("Unknown ISA {!r}.".format(isa)) diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index 6191fbe..01591c3 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -2,28 +2,46 @@ import os import re +from itertools import product from ruamel import yaml -from osaca import utils +from osaca import utils, __version__ from osaca.parser import ParserX86ATT class MachineModel(object): def __init__(self, arch=None, path_to_yaml=None): if not arch and not path_to_yaml: - raise ValueError('Either arch or path_to_yaml required.') - if arch and path_to_yaml: - raise ValueError('Only one of arch and path_to_yaml is allowed.') - self._path = path_to_yaml - self._arch = arch - if arch: - self._arch = arch.lower() - with open(utils.find_file(self._arch+'.yml'), 'r') as f: - self._data = yaml.load(f, Loader=yaml.Loader) - elif path_to_yaml: - with open(self._path, 'r') as f: - self._data = yaml.load(f, Loader=yaml.Loader) + self._data = { + 'osaca_version': str(__version__), + 'micro_architecture': None, + 'arch_code': None, + 'isa': None, + 'ROB_size': None, + 'retired_uOps_per_cycle': None, + 'scheduler_size': None, + 'hidden_loads': None, + 'load_latency': {}, + 'load_throughput': [ + {'base': b, 'index': i, 'offset': o, 'scale': s, 'port_pressure': []} + for b, i, o, s in product(['gpr'], ['gpr', None], ['imd', None], [1, 8])], + 'ports': [], + 'port_model_scheme': None, + 'instruction_forms': [], + } + else: + if arch and path_to_yaml: + raise ValueError('Only one of arch and path_to_yaml is allowed.') + self._path = path_to_yaml + self._arch = arch + if arch: + self._arch = arch.lower() + with open(utils.find_file(self._arch+'.yml'), 'r') as f: + self._data = yaml.load(f, Loader=yaml.Loader) + elif path_to_yaml: + with open(self._path, 'r') as f: + self._data = yaml.load(f, Loader=yaml.Loader) def __getitem__(self, key): """Return configuration entry.""" @@ -36,20 +54,51 @@ class MachineModel(object): ###################################################### def get_instruction(self, name, operands): + """Find and return instruction data from name and operands.""" if name is None: return None try: return next( instruction_form for instruction_form in self._data['instruction_forms'] - if instruction_form['name'].lower() == name.lower() + if instruction_form['name'].upper() == name.upper() and self._match_operands(instruction_form['operands'], operands) ) except StopIteration: return None - except TypeError: + except TypeError as e: print('\nname: {}\noperands: {}'.format(name, operands)) - raise TypeError + raise TypeError from e + + def average_port_pressure(self, port_pressure): + """Construct average port pressure list from instruction data.""" + port_list = self._data['ports'] + average_pressure = [0.0]*len(port_list) + for cycles, ports in port_pressure: + for p in ports: + average_pressure[port_list.index(p)] += cycles/len(ports) + + return average_pressure + + def set_instruction(self, name, operands, + latency=None, port_pressure=None, throughput=None, uops=None): + """Import instruction form information.""" + # If it already exists. Overwrite information. + instr_data = self.get_instruction(name, operands) + if instr_data is None: + instr_data = {} + self._data['instruction_forms'].append(instr_data) + + instr_data['name'] = name + instr_data['operands'] = operands + instr_data['latency'] = latency + instr_data['port_pressure'] = port_pressure + instr_data['throughput'] = throughput + instr_data['uops'] = uops + + def add_port(self, port): + if port not in self._data['ports']: + self._data['ports'].append(port) def get_ISA(self): return self._data['isa'] @@ -103,6 +152,9 @@ class MachineModel(object): return arch_dict[arch].lower() return None + def dump(self): + return yaml.dump(self._data, Dumper=yaml.Dumper, allow_unicode=True) + ###################################################### def _check_for_duplicate(self, name, operands):