Lerking/OSACA
1
0
Fork 0
mirror of https://github.com/RRZE-HPC/OSACA.git synced 2026-01-04 18:20:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

added documentation

Browse Source
This commit is contained in:
JanLJL
2020-03-09 16:35:06 +01:00
parent 458b5954b9
commit 52ca93ad03
4 changed files with 162 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
osaca/semantics/hw_model.py
View File

@@ -118,6 +118,7 @@ class MachineModel(object):
raise TypeError from e
def get_instruction_from_dict(self, name, operands):
"""Find and return instruction data from name and operands stored in dictionary."""
if name is None:
return None
try:
@@ -154,6 +155,7 @@ class MachineModel(object):
instr_data['uops'] = uops
def set_instruction_entry(self, entry):
"""Import instruction as entry object form information."""
self.set_instruction(
entry['name'],
entry['operands'] if 'operands' in entry else None,
@@ -164,55 +166,67 @@ class MachineModel(object):
)
def add_port(self, port):
"""Add port in port model of current machine model."""
if port not in self._data['ports']:
self._data['ports'].append(port)
def get_ISA(self):
"""Return ISA of :class:`MachineModel`."""
return self._data['isa'].lower()
def get_arch(self):
"""Return micro-architecture code of :class:`MachineModel`."""
return self._data['arch_code'].lower()
def get_ports(self):
"""Return port model of :class:`MachineModel`."""
return self._data['ports']
def has_hidden_loads(self):
"""Return if model has hidden loads."""
if 'hidden_loads' in self._data:
return self._data['hidden_loads']
return False
def get_load_latency(self, reg_type):
"""Return load latency for given register type."""
return self._data['load_latency'][reg_type]
def get_load_throughput(self, memory):
"""Return load thorughput for given register type."""
ld_tp = [m for m in self._data['load_throughput'] if self._match_mem_entries(memory, m)]
if len(ld_tp) > 0:
return ld_tp[0]['port_pressure']
return self._data['load_throughput_default']
def get_store_latency(self, reg_type):
"""Return store latency for given register type."""
# assume 0 for now, since load-store-dependencies currently not detectable
return 0
def get_store_throughput(self, memory):
"""Return store throughput for given register type."""
st_tp = [m for m in self._data['store_throughput'] if self._match_mem_entries(memory, m)]
if len(st_tp) > 0:
return st_tp[0]['port_pressure']
return self._data['store_throughput_default']
def _match_mem_entries(self, mem, i_mem):
"""Check if memory addressing ``mem`` and ``i_mem`` are of the same type."""
if self._data['isa'].lower() == 'aarch64':
return self._is_AArch64_mem_type(i_mem, mem)
if self._data['isa'].lower() == 'x86':
return self._is_x86_mem_type(i_mem, mem)
def get_data_ports(self):
"""Return all data ports (i.e., ports with D-suffix) of current model."""
data_port = re.compile(r'^[0-9]+D$')
data_ports = [x for x in filter(data_port.match, self._data['ports'])]
return data_ports
@staticmethod
def get_full_instruction_name(instruction_form):
"""Get one instruction name string including the mnemonic and all operands."""
operands = []
for op in instruction_form['operands']:
op_attrs = [
@@ -224,6 +238,7 @@ class MachineModel(object):
@staticmethod
def get_isa_for_arch(arch):
"""Return ISA for given micro-arch ``arch``."""
arch_dict = {
'tx2': 'aarch64',
'zen1': 'x86',
@@ -252,6 +267,7 @@ class MachineModel(object):
raise ValueError("Unknown architecture {!r}.".format(arch))
def dump(self, stream=None):
"""Dump machine model to stream or return it as a ``str`` if no stream is given."""
# Replace instruction form's port_pressure with styled version for RoundtripDumper
formatted_instruction_forms = deepcopy(self._data['instruction_forms'])
for instruction_form in formatted_instruction_forms:
@@ -289,6 +305,13 @@ class MachineModel(object):
######################################################
def _get_cached(self, filepath):
"""
Check if machine model is cached and if so, load it.
:param filepath: path to check for cached machine model
:type filepath: str
:returns: cached DB if existing, `False` otherwise
"""
hashname = self._get_hashname(filepath)
cachepath = utils.exists_cached_file(hashname + '.pickle')
if cachepath:
@@ -303,14 +326,24 @@ class MachineModel(object):
return False
def _write_in_cache(self, filepath, data):
"""
Write machine model to cache
:param filepath: path to store DB
:type filepath: str
:param data: :class:`MachineModel` to store
:type data: :class:`dict`
"""
hashname = self._get_hashname(filepath)
filepath = os.path.join(utils.CACHE_DIR, hashname + '.pickle')
pickle.dump(data, open(filepath, 'wb'))
def _get_hashname(self, name):
"""Returns unique hashname for machine model"""
return base64.b64encode(name.encode()).decode()
def _get_key(self, name, operands):
"""Get unique instruction form key for dict DB."""
key_string = name.lower() + '-'
if operands is None:
return key_string[:-1]
@@ -318,6 +351,7 @@ class MachineModel(object):
return key_string
def _convert_to_dict(self, instruction_forms):
"""Convert list DB to dict DB"""
instruction_dict = {}
for instruction_form in instruction_forms:
instruction_dict[
@@ -329,6 +363,7 @@ class MachineModel(object):
return instruction_dict
def _get_operand_hash(self, operand):
"""Get unique key for operand for dict DB"""
operand_string = ''
if 'class' in operand:
# DB entry
@@ -362,6 +397,7 @@ class MachineModel(object):
return operand_string
def _create_db_operand_aarch64(operand):
"""Create instruction form operand for DB out of operand string."""
if operand == 'i':
return {'class': 'immediate', 'imd': 'int'}
elif operand in 'wxbhsdq':
@@ -382,6 +418,7 @@ class MachineModel(object):
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
def _create_db_operand_x86(operand):
"""Create instruction form operand for DB out of operand string."""
if operand == 'r':
return {'class': 'register', 'name': 'gpr'}
elif operand in 'xyz':
@@ -400,6 +437,14 @@ class MachineModel(object):
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
def _check_for_duplicate(self, name, operands):
"""
Check if instruction form exists at least twice in DB.
:param str name: mnemonic of instruction form
:param list operands: instruction form operands
:returns: `True`, if duplicate exists, `False` otherwise
"""
matches = [
instruction_form
for instruction_form in self._data['instruction_forms']
@@ -411,6 +456,7 @@ class MachineModel(object):
return False
def _match_operands(self, i_operands, operands):
"""Check if all operand types of ``i_operands`` and ``operands`` match."""
operands_ok = True
if len(operands) != len(i_operands):
return False
@@ -423,6 +469,7 @@ class MachineModel(object):
return False
def _check_operands(self, i_operand, operand):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
# check for wildcard
if self.WILDCARD in operand:
if (
@@ -439,6 +486,7 @@ class MachineModel(object):
return self._check_x86_operands(i_operand, operand)
def _check_AArch64_operands(self, i_operand, operand):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
if 'class' in operand:
# compare two DB entries
return self._compare_db_entries(i_operand, operand)
@@ -470,6 +518,7 @@ class MachineModel(object):
return False
def _check_x86_operands(self, i_operand, operand):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
if 'class' in operand:
# compare two DB entries
return self._compare_db_entries(i_operand, operand)
@@ -491,6 +540,7 @@ class MachineModel(object):
return i_operand['class'] == 'identifier'
def _compare_db_entries(self, operand_1, operand_2):
"""Check if operand types in DB format (i.e., not parsed) match."""
operand_attributes = list(
filter(lambda x: True if x != 'source' and x != 'destination' else False, operand_1)
)
@@ -503,6 +553,7 @@ class MachineModel(object):
return True
def _is_AArch64_reg_type(self, i_reg, reg):
"""Check if register type match."""
if reg['prefix'] != i_reg['prefix']:
return False
if 'shape' in reg:
@@ -512,6 +563,7 @@ class MachineModel(object):
return True
def _is_x86_reg_type(self, i_reg_name, reg):
"""Check if register type match."""
# differentiate between vector registers (mm, xmm, ymm, zmm) and others (gpr)
parser_x86 = ParserX86ATT()
if parser_x86.is_vector_register(reg):
@@ -523,6 +575,7 @@ class MachineModel(object):
return False
def _is_AArch64_mem_type(self, i_mem, mem):
"""Check if memory addressing type match."""
if (
# check base
(
@@ -576,6 +629,7 @@ class MachineModel(object):
return False
def _is_x86_mem_type(self, i_mem, mem):
"""Check if memory addressing type match."""
if (
# check base
(
@@ -627,6 +681,7 @@ class MachineModel(object):
return False
def _create_yaml_object(self):
"""Create YAML object for parsing and dumping DB"""
yaml_obj = ruamel.yaml.YAML()
yaml_obj.representer.add_representer(type(None), self.__represent_none)
yaml_obj.default_flow_style = None
@@ -635,4 +690,5 @@ class MachineModel(object):
return yaml_obj
def __represent_none(self, yaml_obj, data):
"""YAML representation for `None`"""
return yaml_obj.represent_scalar(u'tag:yaml.org,2002:null', u'~')

4
osaca/semantics/isa_semantics.py
View File

@@ -129,6 +129,7 @@ class ISASemantics(object):
instruction_form['flags'] += [INSTR_FLAGS.HAS_ST]
def _has_load(self, instruction_form):
"""Check if instruction form performs a LOAD"""
for operand in chain(
instruction_form['semantic_operands']['source'],
instruction_form['semantic_operands']['src_dst'],
@@ -138,6 +139,7 @@ class ISASemantics(object):
return False
def _has_store(self, instruction_form):
"""Check if instruction form perfroms a STORE"""
for operand in chain(
instruction_form['semantic_operands']['destination'],
instruction_form['semantic_operands']['src_dst'],
@@ -147,6 +149,7 @@ class ISASemantics(object):
return False
def _get_regular_source_operands(self, instruction_form):
"""Get source operand of given instruction form assuming regular src/dst behavior."""
# if there is only one operand, assume it is a source operand
if len(instruction_form['operands']) == 1:
return [instruction_form['operands'][0]]
@@ -159,6 +162,7 @@ class ISASemantics(object):
raise ValueError("Unsupported ISA {}.".format(self._isa))
def _get_regular_destination_operands(self, instruction_form):
"""Get destination operand of given instruction form assuming regular src/dst behavior."""
# if there is only one operand, assume no destination
if len(instruction_form['operands']) == 1:
return []

73
osaca/semantics/kernel_dg.py
View File

@@ -18,6 +18,13 @@ class KernelDG(nx.DiGraph):
self.loopcarried_deps = self.check_for_loopcarried_dep(self.kernel)
def create_DG(self, kernel):
"""
Create directed graph from given kernel
:param kernel: Parsed asm kernel with assigned semantic information
:type kerne: list
:returns: :class:`~nx.DiGraph` -- directed graph object
"""
# 1. go through kernel instruction forms and add them as node attribute
# 2. find edges (to dependend further instruction)
# 3. get LT value and set as edge weight
@@ -41,7 +48,7 @@ class KernelDG(nx.DiGraph):
instruction_form['line_number'],
latency=instruction_form['latency'] - instruction_form['latency_wo_load'],
)
for dep in self.find_depending(instruction_form, kernel[i + 1:]):
for dep in self.find_depending(instruction_form, kernel[i + 1 :]):
edge_weight = (
instruction_form['latency']
if 'latency_wo_load' not in instruction_form
@@ -54,6 +61,13 @@ class KernelDG(nx.DiGraph):
return dg
def check_for_loopcarried_dep(self, kernel):
"""
Try to find loop-carried dependencies in given kernel.
:param kernel: Parsed asm kernel with assigned semantic information
:type kernel: list
:returns: `dict` -- dependency dictionary with all cyclic LCDs
"""
multiplier = len(kernel) + 1
# increase line number for second kernel loop
kernel_length = len(kernel)
@@ -111,9 +125,11 @@ class KernelDG(nx.DiGraph):
return loopcarried_deps_dict
def _get_node_by_lineno(self, lineno):
"""Return instruction form with line number ``lineno`` from kernel"""
return [instr for instr in self.kernel if instr.line_number == lineno][0]
def get_critical_path(self):
"""Find and return critical path after the creation of a directed graph."""
if nx.algorithms.dag.is_directed_acyclic_graph(self.dg):
longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight='latency')
for line_number in longest_path:
@@ -141,17 +157,35 @@ class KernelDG(nx.DiGraph):
raise NotImplementedError('Kernel is cyclic.')
def get_loopcarried_dependencies(self):
"""
Return all LCDs from kernel (after :func:`~KernelDG.check_for_loopcarried_dep` was run)
"""
if nx.algorithms.dag.is_directed_acyclic_graph(self.dg):
return self.loopcarried_deps
else:
# split to DAG
raise NotImplementedError('Kernel is cyclic.')
def find_depending(self, instruction_form, kernel, include_write=False, flag_dependencies=False):
def find_depending(
self, instruction_form, kernel, include_write=False, flag_dependencies=False
):
"""
Find instructions in kernel depending on a given instruction form.
:param dict instruction_form: instruction form to check for dependencies
:param list kernel: kernel containing the instructions to check
:param include_write: indicating if instruction ending the dependency chain should be included, defaults to `False`
:type include_write: boolean, optional
:param flag_dependencies: indicating if dependencies of flags should be considered, defaults to `False`
:type flag_dependencies: boolean, optional
:returns: iterator if all directly dependent instruction forms
"""
if instruction_form.semantic_operands is None:
return
for dst in chain(instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst):
for dst in chain(
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'register' in dst:
# Check for read of register until overwrite
for instr_form in kernel:
@@ -212,11 +246,13 @@ class KernelDG(nx.DiGraph):
return iter([])
def is_read(self, register, instruction_form):
"""Check if instruction form reads from given register"""
is_read = False
if instruction_form.semantic_operands is None:
return is_read
for src in chain(instruction_form.semantic_operands.source,
instruction_form.semantic_operands.src_dst):
for src in chain(
instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst
):
if 'register' in src:
is_read = self.parser.is_reg_dependend_of(register, src.register) or is_read
if 'flag' in src:
@@ -229,8 +265,10 @@ class KernelDG(nx.DiGraph):
self.parser.is_reg_dependend_of(register, src.memory.index) or is_read
)
# Check also if read in destination memory address
for dst in chain(instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst):
for dst in chain(
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'memory' in dst:
if dst.memory.base is not None:
is_read = self.parser.is_reg_dependend_of(register, dst.memory.base) or is_read
@@ -241,11 +279,14 @@ class KernelDG(nx.DiGraph):
return is_read
def is_written(self, register, instruction_form):
"""Check if instruction form writes in given register"""
is_written = False
if instruction_form.semantic_operands is None:
return is_written
for dst in chain(instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst):
for dst in chain(
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'register' in dst:
is_written = self.parser.is_reg_dependend_of(register, dst.register) or is_written
if 'flag' in dst:
@@ -256,8 +297,9 @@ class KernelDG(nx.DiGraph):
self.parser.is_reg_dependend_of(register, dst.memory.base) or is_written
)
# Check also for possible pre- or post-indexing in memory addresses
for src in chain(instruction_form.semantic_operands.source,
instruction_form.semantic_operands.src_dst):
for src in chain(
instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst
):
if 'memory' in src:
if 'pre_indexed' in src.memory or 'post_indexed' in src.memory:
is_written = (
@@ -266,6 +308,13 @@ class KernelDG(nx.DiGraph):
return is_written
def export_graph(self, filepath=None):
"""
Export graph with highlighted CP and LCDs as DOT file. Writes it to 'osaca_dg.dot'
if no other path is given.
:param filepath: path to write DOT file, defaults to None.
:type filepath: str, optional
"""
graph = copy.deepcopy(self.dg)
cp = self.get_critical_path()
cp_line_numbers = [x['line_number'] for x in cp]

42
osaca/semantics/marker_utils.py
View File

@@ -7,6 +7,13 @@ COMMENT_MARKER = {'start': 'OSACA-BEGIN', 'end': 'OSACA-END'}
def reduce_to_section(kernel, isa):
"""
Finds OSACA markers in given kernel and returns marked section
:param list kernel: kernel to check
:param str isa: ISA of given kernel
:returns: `list` -- marked section of kernel as list of instruction forms
"""
isa = isa.lower()
if isa == 'x86':
start, end = find_marked_kernel_x86ATT(kernel)
@@ -22,6 +29,12 @@ def reduce_to_section(kernel, isa):
def find_marked_kernel_AArch64(lines):
"""
Find marked section for AArch64
:param list lines: kernel
:returns: `tuple of int` -- start and end line of marked section
"""
nop_bytes = ['213', '3', '32', '31']
return find_marked_section(
lines,
@@ -36,6 +49,12 @@ def find_marked_kernel_AArch64(lines):
def find_marked_kernel_x86ATT(lines):
"""
Find marked section for x86
:param list lines: kernel
:returns: `tuple of int` -- start and end line of marked section
"""
nop_bytes = ['100', '103', '144']
return find_marked_section(
lines,
@@ -59,7 +78,7 @@ def get_marker(isa, comment=""):
'.byte 144 # OSACA START MARKER\n'
)
if comment:
start_marker_raw += "# {}\m".format(comment)
start_marker_raw += "# {}\n".format(comment)
end_marker_raw = (
'movl $222, %ebx # OSACA END MARKER\n'
'.byte 100 # OSACA END MARKER\n'
@@ -89,6 +108,26 @@ def get_marker(isa, comment=""):
def find_marked_section(
lines, parser, mov_instr, mov_reg, mov_vals, nop_bytes, reverse=False, comments=None
):
"""
Return indexes of marked section
:param list lines: kernel
:param parser: parser to use for checking
:type parser: :class:`~parser.BaseParser`
:param mov_instr: all MOV instruction possible for the marker
:type mov_instr: `list of str`
:param mov_reg: register used for the marker
:type mov_reg: `str`
:param mov_vals: values needed to be moved to ``mov_reg`` for valid marker
:type mov_vals: `list of int`
:param nop_bytes: bytes representing opcode of NOP
:type nop_bytes: `list of int`
:param reverse: indicating if ISA syntax requires reverse operand order, defaults to `False`
:type reverse: boolean, optional
:param comments: dictionary with start and end markers in comment format, defaults to None
:type comments: dict, optional
:returns: `tuple of int` -- start and end line of marked section
"""
# TODO match to instructions returned by get_marker
index_start = -1
index_end = -1
@@ -133,6 +172,7 @@ def find_marked_section(
def match_bytes(lines, index, byte_list):
"""Match bytes directives of markers"""
# either all bytes are in one line or in separate ones
extracted_bytes = []
line_count = 0