mirror of
https://github.com/RRZE-HPC/OSACA.git
synced 2026-01-09 04:30:05 +01:00
work in progress
This commit is contained in:
Julian Hammer
@@ -4,115 +4,29 @@ import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from distutils.version import StrictVersion
|
||||
from itertools import groupby, product
|
||||
|
||||
from ruamel import yaml
|
||||
|
||||
from osaca.db_interface import add_entries_to_db
|
||||
from osaca.parser import ParserAArch64v81, ParserX86ATT
|
||||
from osaca.parser import get_parser
|
||||
from osaca.semantics import MachineModel
|
||||
|
||||
ARCH_DICT = {
|
||||
'vulcan': 'aarch64',
|
||||
'snb': 'x86',
|
||||
'ivb': 'x86',
|
||||
'hsw': 'x86',
|
||||
'bdw': 'x86',
|
||||
'skl': 'x86',
|
||||
'skx': 'x86',
|
||||
'csx': 'x86',
|
||||
}
|
||||
|
||||
def port_pressure_from_tag_attributes(attrib):
|
||||
# '1*p015+1*p1+1*p23+1*p4+3*p5' ->
|
||||
# [(1, '015'), (1, '1'), (1, '23'), (1, '4'), (3, '5')]
|
||||
port_occupation = []
|
||||
for p in attrib['ports'].split('+'):
|
||||
cycles, ports = p.split('*p')
|
||||
port_occupation.append((int(cycles), ports))
|
||||
|
||||
def port_pressure_from_tag_attributes(attrib, arch, ports):
|
||||
# apply cycles for D ports
|
||||
data_port = re.compile(r'[0-9]D$')
|
||||
data_ports = [x[:-1] for x in filter(data_port.match, ports)]
|
||||
|
||||
# format attributes
|
||||
cycles = attrib['ports'].split('+')
|
||||
cycles = [c.split('*') for c in cycles]
|
||||
for i, c in enumerate(cycles):
|
||||
cycles[i][0] = int(c[0])
|
||||
if str(c[1]).startswith('p'):
|
||||
cycles[i][1] = [p for p in c[1][1:]]
|
||||
if data_ports and data_ports == cycles[i][1]:
|
||||
# uops for data ports
|
||||
cycles.append([c[0], [x + 'D' for x in data_ports]])
|
||||
cycles[i][0] = [
|
||||
cycles[i][0] / num for num in range(1, len(cycles[i][1]) + 1) for _ in range(num)
|
||||
]
|
||||
cycles = [list(product(c[0], c[1])) for c in cycles]
|
||||
all_options = []
|
||||
|
||||
# iterate over all combinations of all uop options
|
||||
for cycles_combs in cycles:
|
||||
options = []
|
||||
tmp_opt = []
|
||||
total = cycles_combs[0][0]
|
||||
# iterate over all combinations of each uop option
|
||||
for comb in cycles_combs:
|
||||
# add options until they reach the total num of uops
|
||||
tmp_opt.append(comb)
|
||||
if sum([c[0] for c in tmp_opt]) == total:
|
||||
# copy this option as one of several to the cycle option list
|
||||
options.append(tmp_opt.copy())
|
||||
tmp_opt = []
|
||||
if len(tmp_opt) != 0:
|
||||
raise ValueError('Cannot compute port pressure')
|
||||
options = [x for x, _ in groupby(options)]
|
||||
all_options.append(options)
|
||||
all_options = list(product(*all_options))
|
||||
|
||||
# find best scheduling
|
||||
port_pressure = {}
|
||||
for p in ports:
|
||||
port_pressure[p] = 0.0
|
||||
first = calculate_port_pressure(all_options[0])
|
||||
for key in first:
|
||||
port_pressure[key] = first[key]
|
||||
for option in all_options[1:]:
|
||||
tmp = calculate_port_pressure(option)
|
||||
if (max(list(tmp.values())) <= max(list(port_pressure.values()))) and (
|
||||
len(tmp) > len([x for x in port_pressure.values() if x != 0.0])
|
||||
):
|
||||
for k in port_pressure:
|
||||
port_pressure[k] = tmp[k] if k in tmp else 0.0
|
||||
|
||||
# check if calculation equals given throughput
|
||||
if abs(max(list(port_pressure.values())) - float(attrib['TP_ports'])) > 0.01:
|
||||
print('Contradicting TP value compared to port_pressure. Ignore port pressure.')
|
||||
for p in port_pressure:
|
||||
port_pressure[p] = 0.0
|
||||
return port_pressure
|
||||
|
||||
# Also consider DIV pipeline
|
||||
# Also
|
||||
if 'div_cycles' in attrib:
|
||||
div_port = re.compile(r'[0-9]DV$')
|
||||
div_ports = [x for x in filter(div_port.match, ports)]
|
||||
for dp in div_ports:
|
||||
port_pressure[dp] += int(attrib['div_cycles']) / len(div_ports)
|
||||
return port_pressure
|
||||
port_occupation.append((int(attrib['div_cycles']), ('DV',)))
|
||||
|
||||
return port_occupation
|
||||
|
||||
|
||||
def calculate_port_pressure(pp_option):
|
||||
ports = {}
|
||||
for option in pp_option:
|
||||
for port in option:
|
||||
if port[1] in ports:
|
||||
ports[port[1]] += port[0]
|
||||
else:
|
||||
ports[port[1]] = port[0]
|
||||
return ports
|
||||
|
||||
|
||||
def extract_paramters(instruction_tag, arch):
|
||||
isa = ARCH_DICT[arch.lower()]
|
||||
parser = ParserX86ATT()
|
||||
if isa == 'aarch64':
|
||||
parser = ParserAArch64v81()
|
||||
elif isa == 'x86':
|
||||
parser = ParserX86ATT()
|
||||
def extract_paramters(instruction_tag, isa):
|
||||
parser = get_parser(isa)
|
||||
# Extract parameter components
|
||||
parameters = [] # used to store string representations
|
||||
parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib['idx']))
|
||||
@@ -138,16 +52,12 @@ def extract_paramters(instruction_tag, arch):
|
||||
parameter['class'] = 'register'
|
||||
possible_regs = [parser.parse_register('%' + r) for r in parameter_tag.text.split(',')]
|
||||
if possible_regs[0] is None:
|
||||
raise ValueError(
|
||||
'Unknown register type for {} with {}.'.format(
|
||||
parameter_tag.attrib, parameter_tag.text
|
||||
)
|
||||
)
|
||||
raise ValueError('Unknown register type for {} with {}.'.format(
|
||||
parameter_tag.attrib, parameter_tag.text))
|
||||
if isa == 'x86':
|
||||
if parser.is_vector_register(possible_regs[0]['register']):
|
||||
possible_regs[0]['register']['name'] = possible_regs[0]['register'][
|
||||
'name'
|
||||
].lower()[:3]
|
||||
possible_regs[0]['register']['name'] = \
|
||||
possible_regs[0]['register']['name'].lower()[:3]
|
||||
if 'mask' in possible_regs[0]['register']:
|
||||
possible_regs[0]['register']['mask'] = True
|
||||
else:
|
||||
@@ -175,9 +85,8 @@ def extract_paramters(instruction_tag, arch):
|
||||
|
||||
|
||||
def extract_model(tree, arch):
|
||||
mm = MachineModel(arch.lower())
|
||||
ports = mm._data['ports']
|
||||
model_data = []
|
||||
mm = MachineModel()
|
||||
|
||||
for instruction_tag in tree.findall('.//instruction'):
|
||||
ignore = False
|
||||
|
||||
@@ -185,8 +94,8 @@ def extract_model(tree, arch):
|
||||
|
||||
# Extract parameter components
|
||||
try:
|
||||
parameters = extract_paramters(instruction_tag, arch)
|
||||
if ARCH_DICT[arch.lower()] == 'x86':
|
||||
parameters = extract_paramters(instruction_tag, mm.get_isa_for_arch(arch))
|
||||
if mm.get_isa_for_arch(arch).lower() == 'x86':
|
||||
parameters.reverse()
|
||||
except ValueError as e:
|
||||
print(e, file=sys.stderr)
|
||||
@@ -201,16 +110,12 @@ def extract_model(tree, arch):
|
||||
if 'TP_ports' in measurement_tag.attrib:
|
||||
throughput = measurement_tag.attrib['TP_ports']
|
||||
else:
|
||||
throughput = (
|
||||
measurement_tag.attrib['TP'] if 'TP' in measurement_tag.attrib else None
|
||||
)
|
||||
uops = (
|
||||
int(measurement_tag.attrib['uops']) if 'uops' in measurement_tag.attrib else None
|
||||
)
|
||||
throughput = (measurement_tag.attrib['TP']
|
||||
if 'TP' in measurement_tag.attrib else None)
|
||||
uops = (int(measurement_tag.attrib['uops'])
|
||||
if 'uops' in measurement_tag.attrib else None)
|
||||
if 'ports' in measurement_tag.attrib:
|
||||
port_pressure.append(
|
||||
port_pressure_from_tag_attributes(measurement_tag.attrib, arch, ports)
|
||||
)
|
||||
port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib))
|
||||
latencies = [
|
||||
int(l_tag.attrib['cycles'])
|
||||
for l_tag in measurement_tag.iter('latency')
|
||||
@@ -230,39 +135,30 @@ def extract_model(tree, arch):
|
||||
|
||||
# Ordered by IACA version (newest last)
|
||||
for iaca_tag in sorted(
|
||||
arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version'])
|
||||
):
|
||||
arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version'])):
|
||||
if 'ports' in iaca_tag.attrib:
|
||||
port_pressure.append(
|
||||
port_pressure_from_tag_attributes(iaca_tag.attrib, arch, ports)
|
||||
)
|
||||
port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib))
|
||||
if ignore:
|
||||
continue
|
||||
|
||||
# Add missing ports:
|
||||
[p[1] for p in for pp in port_pressure]
|
||||
mm.add_port()
|
||||
|
||||
# Check if all are equal
|
||||
if port_pressure:
|
||||
if port_pressure[1:] != port_pressure[:-1]:
|
||||
print(
|
||||
"Contradicting port occupancies, using latest IACA:", mnemonic, file=sys.stderr
|
||||
)
|
||||
print("Contradicting port occupancies, using latest IACA:", mnemonic,
|
||||
file=sys.stderr)
|
||||
port_pressure = port_pressure[-1]
|
||||
throughput = max(list(port_pressure.values()) + [0.0])
|
||||
throughput = max(mm.average_port_pressure(port_pressure))
|
||||
else:
|
||||
# print("No data available for this architecture:", mnemonic, file=sys.stderr)
|
||||
continue
|
||||
# ---------------------------------------------
|
||||
model_data.append(
|
||||
{
|
||||
'name': mnemonic,
|
||||
'operands': parameters,
|
||||
'uops': uops,
|
||||
'throughput': throughput,
|
||||
'latency': latency,
|
||||
'port_pressure': port_pressure,
|
||||
}
|
||||
)
|
||||
mm.set_instruction(mnemonic, parameters, latency, port_pressure, throughput, uops)
|
||||
|
||||
return model_data
|
||||
return mm
|
||||
|
||||
|
||||
def architectures(tree):
|
||||
@@ -282,12 +178,10 @@ def main():
|
||||
|
||||
tree = ET.parse(args.xml)
|
||||
if args.arch:
|
||||
model_data = extract_model(tree, args.arch)
|
||||
print(yaml.dump(model_data, allow_unicode=True))
|
||||
model = extract_model(tree, args.arch)
|
||||
print(model.dump())
|
||||
else:
|
||||
for arch in architectures(tree):
|
||||
model_data = extract_model(tree, arch)
|
||||
add_entries_to_db(arch, model_data)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -8,4 +8,12 @@ from .base_parser import BaseParser
|
||||
from .parser_x86att import ParserX86ATT
|
||||
from .parser_AArch64v81 import ParserAArch64v81
|
||||
|
||||
__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64v81']
|
||||
__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64v81', 'get_parser']
|
||||
|
||||
def get_parser(isa):
|
||||
if isa.lower() == 'x86':
|
||||
return ParserX86ATT()
|
||||
elif isa.lower() == 'aarch64':
|
||||
return ParserAArch64v81()
|
||||
else:
|
||||
raise ValueError("Unknown ISA {!r}.".format(isa))
|
||||
|
||||
@@ -2,28 +2,46 @@
|
||||
|
||||
import os
|
||||
import re
|
||||
from itertools import product
|
||||
|
||||
from ruamel import yaml
|
||||
|
||||
from osaca import utils
|
||||
from osaca import utils, __version__
|
||||
from osaca.parser import ParserX86ATT
|
||||
|
||||
|
||||
class MachineModel(object):
|
||||
def __init__(self, arch=None, path_to_yaml=None):
|
||||
if not arch and not path_to_yaml:
|
||||
raise ValueError('Either arch or path_to_yaml required.')
|
||||
if arch and path_to_yaml:
|
||||
raise ValueError('Only one of arch and path_to_yaml is allowed.')
|
||||
self._path = path_to_yaml
|
||||
self._arch = arch
|
||||
if arch:
|
||||
self._arch = arch.lower()
|
||||
with open(utils.find_file(self._arch+'.yml'), 'r') as f:
|
||||
self._data = yaml.load(f, Loader=yaml.Loader)
|
||||
elif path_to_yaml:
|
||||
with open(self._path, 'r') as f:
|
||||
self._data = yaml.load(f, Loader=yaml.Loader)
|
||||
self._data = {
|
||||
'osaca_version': str(__version__),
|
||||
'micro_architecture': None,
|
||||
'arch_code': None,
|
||||
'isa': None,
|
||||
'ROB_size': None,
|
||||
'retired_uOps_per_cycle': None,
|
||||
'scheduler_size': None,
|
||||
'hidden_loads': None,
|
||||
'load_latency': {},
|
||||
'load_throughput': [
|
||||
{'base': b, 'index': i, 'offset': o, 'scale': s, 'port_pressure': []}
|
||||
for b, i, o, s in product(['gpr'], ['gpr', None], ['imd', None], [1, 8])],
|
||||
'ports': [],
|
||||
'port_model_scheme': None,
|
||||
'instruction_forms': [],
|
||||
}
|
||||
else:
|
||||
if arch and path_to_yaml:
|
||||
raise ValueError('Only one of arch and path_to_yaml is allowed.')
|
||||
self._path = path_to_yaml
|
||||
self._arch = arch
|
||||
if arch:
|
||||
self._arch = arch.lower()
|
||||
with open(utils.find_file(self._arch+'.yml'), 'r') as f:
|
||||
self._data = yaml.load(f, Loader=yaml.Loader)
|
||||
elif path_to_yaml:
|
||||
with open(self._path, 'r') as f:
|
||||
self._data = yaml.load(f, Loader=yaml.Loader)
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""Return configuration entry."""
|
||||
@@ -36,20 +54,51 @@ class MachineModel(object):
|
||||
######################################################
|
||||
|
||||
def get_instruction(self, name, operands):
|
||||
"""Find and return instruction data from name and operands."""
|
||||
if name is None:
|
||||
return None
|
||||
try:
|
||||
return next(
|
||||
instruction_form
|
||||
for instruction_form in self._data['instruction_forms']
|
||||
if instruction_form['name'].lower() == name.lower()
|
||||
if instruction_form['name'].upper() == name.upper()
|
||||
and self._match_operands(instruction_form['operands'], operands)
|
||||
)
|
||||
except StopIteration:
|
||||
return None
|
||||
except TypeError:
|
||||
except TypeError as e:
|
||||
print('\nname: {}\noperands: {}'.format(name, operands))
|
||||
raise TypeError
|
||||
raise TypeError from e
|
||||
|
||||
def average_port_pressure(self, port_pressure):
|
||||
"""Construct average port pressure list from instruction data."""
|
||||
port_list = self._data['ports']
|
||||
average_pressure = [0.0]*len(port_list)
|
||||
for cycles, ports in port_pressure:
|
||||
for p in ports:
|
||||
average_pressure[port_list.index(p)] += cycles/len(ports)
|
||||
|
||||
return average_pressure
|
||||
|
||||
def set_instruction(self, name, operands,
|
||||
latency=None, port_pressure=None, throughput=None, uops=None):
|
||||
"""Import instruction form information."""
|
||||
# If it already exists. Overwrite information.
|
||||
instr_data = self.get_instruction(name, operands)
|
||||
if instr_data is None:
|
||||
instr_data = {}
|
||||
self._data['instruction_forms'].append(instr_data)
|
||||
|
||||
instr_data['name'] = name
|
||||
instr_data['operands'] = operands
|
||||
instr_data['latency'] = latency
|
||||
instr_data['port_pressure'] = port_pressure
|
||||
instr_data['throughput'] = throughput
|
||||
instr_data['uops'] = uops
|
||||
|
||||
def add_port(self, port):
|
||||
if port not in self._data['ports']:
|
||||
self._data['ports'].append(port)
|
||||
|
||||
def get_ISA(self):
|
||||
return self._data['isa']
|
||||
@@ -103,6 +152,9 @@ class MachineModel(object):
|
||||
return arch_dict[arch].lower()
|
||||
return None
|
||||
|
||||
def dump(self):
|
||||
return yaml.dump(self._data, Dumper=yaml.Dumper, allow_unicode=True)
|
||||
|
||||
######################################################
|
||||
|
||||
def _check_for_duplicate(self, name, operands):
|
||||
|
||||
Reference in New Issue
Block a user