diff --git a/osaca/frontend.py b/osaca/frontend.py new file mode 100755 index 0000000..11e20cb --- /dev/null +++ b/osaca/frontend.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import os +from functools import reduce + +from ruamel import yaml + +from osaca.semantics import INSTR_FLAGS + + +class Frontend(object): + def __init__(self, arch=None, path_to_yaml=None): + if not arch and not path_to_yaml: + raise ValueError('Either arch or path_to_yaml required.') + if arch and path_to_yaml: + raise ValueError('Only one of arch and path_to_yaml is allowed.') + self._arch = arch + if arch: + self._arch = arch.lower() + try: + with open(self._find_file(self._arch), 'r') as f: + self._data = yaml.load(f, Loader=yaml.Loader) + except AssertionError: + raise ValueError( + 'Cannot find specified architecture. Make sure the machine file exists.' + ) + elif path_to_yaml: + try: + with open(self._path, 'r') as f: + self._data = yaml.load(f, Loader=yaml.Loader) + except AssertionError: + raise ValueError( + 'Cannot find specified path to YAML file. Make sure the machine file exists.' + ) + + def _find_file(self, name): + data_dir = os.path.expanduser('~/.osaca/data') + name = os.path.join(data_dir, name + '.yml') + assert os.path.exists(name) + return name + + def _is_comment(self, instruction_form): + return instruction_form['comment'] is not None and instruction_form['instruction'] is None + + def print_throughput_analysis(self, kernel, show_lineno=False, show_cmnts=True): + print() + lineno_filler = ' ' if show_lineno else '' + port_len = self._get_max_port_len(kernel) + separator = '-' * sum([x + 3 for x in port_len]) + '-' + separator += '--' + len(str(kernel[-1]['line_number'])) * '-' if show_lineno else '' + print(lineno_filler + self._get_port_number_line(port_len)) + print(separator) + for instruction_form in kernel: + line = '{:4d} {} {} {}'.format( + instruction_form['line_number'], + self._get_port_pressure(instruction_form['port_pressure'], port_len), + self._get_flag_symbols(instruction_form['flags']) + if instruction_form['instruction'] is not None + else ' ', + instruction_form['line'].strip(), + ) + line = line if show_lineno else '|' + '|'.join(line.split('|')[1:]) + if show_cmnts is False and self._is_comment(instruction_form): + continue + print(line) + print() + tp_sum = reduce( + (lambda x, y: [sum(z) for z in zip(x, y)]), + [instr['port_pressure'] for instr in kernel], + ) + tp_sum = [round(x, 2) for x in tp_sum] + print(lineno_filler + self._get_port_pressure(tp_sum, port_len, ' ')) + + def _get_flag_symbols(self, flag_obj): + string_result = '' + string_result += '*' if INSTR_FLAGS.NOT_BOUND in flag_obj else '' + string_result += 'X' if INSTR_FLAGS.TP_UNKWN in flag_obj else '' + # TODO add other flags + string_result += ' ' if len(string_result) == 0 else '' + return string_result + + def _get_port_pressure(self, ports, port_len, separator='|'): + string_result = '{} '.format(separator) + for i in range(len(ports)): + if float(ports[i]) == 0.0: + string_result += port_len[i] * ' ' + ' {} '.format(separator) + continue + left_len = len(str(float(ports[i])).split('.')[0]) + substr = '{:' + str(left_len) + '.' + str(max(port_len[i] - left_len - 1, 0)) + 'f}' + string_result += substr.format(ports[i]) + ' {} '.format(separator) + return string_result[:-1] + + def _get_max_port_len(self, kernel): + port_len = [4 for x in self._data['ports']] + for instruction_form in kernel: + for i, port in enumerate(instruction_form['port_pressure']): + if len('{:.2f}'.format(port)) > port_len[i]: + port_len[i] = len('{:.2f}'.format(port)) + return port_len + + def _get_port_number_line(self, port_len): + string_result = '|' + for i, length in enumerate(port_len): + substr = '{:^' + str(length + 2) + 's}' + string_result += substr.format(self._data['ports'][i]) + '|' + return string_result + + def print_latency_analysis(self, cp_kernel): + print('\n\n------------------------') + for instruction_form in cp_kernel: + print( + '{} | {} |{}| {}'.format( + instruction_form['line_number'], + instruction_form['latency'], + 'X' if INSTR_FLAGS.LT_UNKWN in instruction_form['flags'] else ' ', + instruction_form['line'], + ) + ) + + def print_list_summary(self): + raise NotImplementedError + + def _print_header_throughput_report(self): + raise NotImplementedError + + def _print_port_binding_summary(self): + raise NotImplementedError diff --git a/osaca/parser/base_parser.py b/osaca/parser/base_parser.py index d2821dd..577d319 100755 --- a/osaca/parser/base_parser.py +++ b/osaca/parser/base_parser.py @@ -28,7 +28,7 @@ class BaseParser(object): asm_instructions = [] lines = file_content.split('\n') for i, line in enumerate(lines): - if line == '': + if line.strip() == '': continue asm_instructions.append(self.parse_line(line, i + 1 + start_line)) return asm_instructions diff --git a/osaca/semantics/__init__.py b/osaca/semantics/__init__.py index 5e6801f..39f8704 100644 --- a/osaca/semantics/__init__.py +++ b/osaca/semantics/__init__.py @@ -6,6 +6,6 @@ Only the classes below will be exported, so please add new semantic tools to __a from .hw_model import MachineModel from .kernel_dg import KernelDG from .marker_utils import reduce_to_section -from .semanticsAppender import SemanticsAppender +from .semanticsAppender import SemanticsAppender, INSTR_FLAGS -__all__ = ['MachineModel', 'KernelDG', 'reduce_to_section', 'SemanticsAppender'] +__all__ = ['MachineModel', 'KernelDG', 'reduce_to_section', 'SemanticsAppender', 'INSTR_FLAGS'] diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index bd27ccb..835b09a 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -69,6 +69,8 @@ class MachineModel(object): ###################################################### def _match_operands(self, i_operands, operands): + if isinstance(operands, dict): + operands = operands['operand_list'] operands_ok = True if len(operands) != len(i_operands): return False @@ -98,13 +100,15 @@ class MachineModel(object): return False return self._is_AArch64_mem_type(i_operand, operand['memory']) # immediate - if 'value' in operand: + if 'value' in operand or ('immediate' in operand and 'value' in operand['immediate']): return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int' - if 'float' in operand: + if 'float' in operand or ('immediate' in operand and 'float' in operand['immediate']): return i_operand['class'] == 'immediate' and i_operand['imd'] == 'float' - if 'double' in operand: + if 'double' in operand or ('immediate' in operand and 'double' in operand['immediate']): return i_operand['class'] == 'immediate' and i_operand['imd'] == 'double' - if 'identifier' in operand: + if 'identifier' in operand or ( + 'immediate' in operand and 'identifier' in operand['immediate'] + ): return i_operand['class'] == 'identifier' # prefetch option if 'prfop' in operand: @@ -124,7 +128,7 @@ class MachineModel(object): return False return self._is_x86_mem_type(i_operand, operand['memory']) # immediate - if 'value' in operand: + if 'immediate' in operand or 'value' in operand: return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int' # identifier (e.g., labels) if 'identifier' in operand: diff --git a/osaca/semantics/kernel_dg.py b/osaca/semantics/kernel_dg.py index 32102cf..f9f6915 100755 --- a/osaca/semantics/kernel_dg.py +++ b/osaca/semantics/kernel_dg.py @@ -32,6 +32,14 @@ class KernelDG(nx.DiGraph): dg.nodes[dep['line_number']]['instruction_form'] = dep return dg + def get_critical_path(self): + if nx.algorithms.dag.is_directed_acyclic_graph(self.dg): + longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight='latency') + return [x for x in self.kernel if x['line_number'] in longest_path] + else: + # split to DAG + raise NotImplementedError + def find_depending(self, instruction_form, kernel): if instruction_form.operands is None: return @@ -72,6 +80,8 @@ class KernelDG(nx.DiGraph): def is_read(self, register, instruction_form): is_read = False + if instruction_form.operands is None: + return is_read for src in instruction_form.operands.source + instruction_form.operands.src_dst: if 'register' in src: is_read = self.parser.is_reg_dependend_of(register, src.register) or is_read @@ -95,6 +105,8 @@ class KernelDG(nx.DiGraph): def is_written(self, register, instruction_form): is_written = False + if instruction_form.operands is None: + return is_written for dst in instruction_form.operands.destination + instruction_form.operands.src_dst: if 'register' in dst: is_written = self.parser.is_reg_dependend_of(register, dst.register) or is_written diff --git a/osaca/semantics/semanticsAppender.py b/osaca/semantics/semanticsAppender.py index 204a047..9150178 100755 --- a/osaca/semantics/semanticsAppender.py +++ b/osaca/semantics/semanticsAppender.py @@ -8,18 +8,20 @@ from osaca.parser import AttrDict from .hw_model import MachineModel -class SemanticsAppender(object): - class INSTR_FLAGS: - """ - Flags used for unknown or special instructions - """ - TP_UNKWN = 'tp_unkown' - LT_UNKWN = 'lt_unkown' - NOT_BOUND = 'not_bound' - HIDDEN_LD = 'hidden_load' +class INSTR_FLAGS: + """ + Flags used for unknown or special instructions + """ + TP_UNKWN = 'tp_unkown' + LT_UNKWN = 'lt_unkown' + NOT_BOUND = 'not_bound' + HIDDEN_LD = 'hidden_load' + + +class SemanticsAppender(object): def __init__(self, machine_model: MachineModel, path_to_yaml=None): - self.machine_model = machine_model + self._machine_model = machine_model self._isa = machine_model.get_ISA() if path_to_yaml: path = path_to_yaml @@ -37,8 +39,8 @@ class SemanticsAppender(object): # mark instruction form with semantic flags def assign_tp_lt(self, instruction_form): flags = [] - port_number = len(self.machine_model['ports']) - instruction_data = self.machine_model.get_instruction( + port_number = len(self._machine_model['ports']) + instruction_data = self._machine_model.get_instruction( instruction_form['instruction'], instruction_form['operands'] ) if instruction_data: @@ -49,6 +51,9 @@ class SemanticsAppender(object): assert isinstance(port_pressure, list) assert len(port_pressure) == port_number instruction_form['port_pressure'] = port_pressure + if sum(port_pressure) == 0: + # port pressure on all ports 0 --> not bound to a port + flags.append(INSTR_FLAGS.NOT_BOUND) except AssertionError: warnings.warn( 'Port pressure could not be imported correctly from database. ' @@ -64,14 +69,14 @@ class SemanticsAppender(object): if latency is None: # assume 0 cy and mark as unknown latency = 0.0 - flags.append(self.INSTR_FLAGS.LT_UNKWN) + flags.append(INSTR_FLAGS.LT_UNKWN) else: # instruction could not be found in DB # --> mark as unknown and assume 0 cy for latency/throughput throughput = 0.0 latency = 0.0 instruction_form['port_pressure'] = [0.0 for i in range(port_number)] - flags += [self.INSTR_FLAGS.TP_UNKWN, self.INSTR_FLAGS.LT_UNKWN] + flags += [INSTR_FLAGS.TP_UNKWN, INSTR_FLAGS.LT_UNKWN] # flatten flag list flags = list(set(flags)) if 'flags' not in instruction_form: diff --git a/setup.py b/setup.py index 31fc8b2..88f68d8 100755 --- a/setup.py +++ b/setup.py @@ -92,7 +92,6 @@ setup( 'kerncraft', 'networkx', 'numpy', - 'pandas', 'pyparsing', ], python_requires='>=3.5', diff --git a/tests/test_semantics.py b/tests/test_semantics.py index f04f560..7a4da43 100755 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -7,6 +7,7 @@ import networkx as nx import os import unittest +from osaca.frontend import Frontend from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT from osaca.semantics.hw_model import MachineModel from osaca.semantics.kernel_dg import KernelDG @@ -109,6 +110,10 @@ class TestSemanticTools(unittest.TestCase): self.assertEqual(len(list(dg.get_dependent_instruction_forms(line_number=6))), 0) self.assertEqual(len(list(dg.get_dependent_instruction_forms(line_number=7))), 0) + fe = Frontend(arch='CSL') + fe.print_throughput_analysis(self.kernel_x86) + fe.print_latency_analysis(dg.get_critical_path()) + def test_kernelDG_AArch64(self): dg = KernelDG(self.kernel_AArch64, self.parser_AArch64, self.machine_model_tx2) self.assertTrue(nx.algorithms.dag.is_directed_acyclic_graph(dg.dg)) @@ -131,6 +136,10 @@ class TestSemanticTools(unittest.TestCase): self.assertEqual(len(list(dg.get_dependent_instruction_forms(line_number=18))), 0) self.assertEqual(len(list(dg.get_dependent_instruction_forms(line_number=19))), 0) + fe = Frontend(arch='vulcan') + fe.print_throughput_analysis(self.kernel_AArch64) + fe.print_latency_analysis(dg.get_critical_path()) + def test_is_read_is_written_x86(self): # independent form HW model dag = KernelDG(self.kernel_x86, self.parser_x86, None)