From 0f5d3a0370b3795dacf3692962eb6d4f75d3b91a Mon Sep 17 00:00:00 2001 From: JanLJL Date: Tue, 29 Oct 2019 09:09:52 +0100 Subject: [PATCH] separated SemanticsAppender into ISA and Arch semantics --- osaca/api/kerncraft_interface.py | 4 +- osaca/frontend.py | 6 +- osaca/osaca.py | 4 +- osaca/semantics/__init__.py | 12 +- ...emantics_appender.py => arch_semantics.py} | 112 +---------------- osaca/semantics/isa_semantics.py | 116 ++++++++++++++++++ osaca/semantics/marker_utils.py | 31 +++-- tests/test_frontend.py | 8 +- tests/test_semantics.py | 10 +- 9 files changed, 167 insertions(+), 136 deletions(-) rename osaca/semantics/{semantics_appender.py => arch_semantics.py} (71%) create mode 100755 osaca/semantics/isa_semantics.py diff --git a/osaca/api/kerncraft_interface.py b/osaca/api/kerncraft_interface.py index e31ddac..47a8d69 100755 --- a/osaca/api/kerncraft_interface.py +++ b/osaca/api/kerncraft_interface.py @@ -7,7 +7,7 @@ from io import StringIO from osaca.frontend import Frontend from osaca.parser import ParserAArch64v81, ParserX86ATT from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel, - SemanticsAppender, reduce_to_section) + ArchSemantics, reduce_to_section) # Stolen from https://stackoverflow.com/a/16571630 @@ -26,7 +26,7 @@ class Capturing(list): class KerncraftAPI(object): def __init__(self, arch, code): self.machine_model = MachineModel(arch=arch) - self.semantics = SemanticsAppender(self.machine_model) + self.semantics = ArchSemantics(self.machine_model) isa = self.machine_model.get_ISA().lower() if isa == 'aarch64': self.parser = ParserAArch64v81() diff --git a/osaca/frontend.py b/osaca/frontend.py index 845c266..fdd2557 100755 --- a/osaca/frontend.py +++ b/osaca/frontend.py @@ -6,7 +6,7 @@ from datetime import datetime as dt from ruamel import yaml from osaca import utils -from osaca.semantics import INSTR_FLAGS, KernelDG, SemanticsAppender +from osaca.semantics import INSTR_FLAGS, KernelDG, ArchSemantics class Frontend(object): @@ -56,7 +56,7 @@ class Frontend(object): continue print(line) print() - tp_sum = SemanticsAppender.get_throughput_sum(kernel) + tp_sum = ArchSemantics.get_throughput_sum(kernel) print(lineno_filler + self._get_port_pressure(tp_sum, port_len, ' ')) def print_latency_analysis(self, cp_kernel, separator='|'): @@ -165,7 +165,7 @@ class Frontend(object): print(line) print() # lcd_sum already calculated before - tp_sum = SemanticsAppender.get_throughput_sum(kernel) + tp_sum = ArchSemantics.get_throughput_sum(kernel) cp_sum = sum([x['latency_cp'] for x in cp_kernel]) print( lineno_filler diff --git a/osaca/osaca.py b/osaca/osaca.py index 71c832e..e95c7d1 100755 --- a/osaca/osaca.py +++ b/osaca/osaca.py @@ -10,7 +10,7 @@ from subprocess import call from osaca.db_interface import sanity_check, import_benchmark_output from osaca.frontend import Frontend from osaca.parser import BaseParser, ParserAArch64v81, ParserX86ATT -from osaca.semantics import (KernelDG, MachineModel, SemanticsAppender, +from osaca.semantics import (KernelDG, MachineModel, ArchSemantics, reduce_to_section) MODULE_DATA_DIR = os.path.join( @@ -181,7 +181,7 @@ def inspect(args): # Reduce to marked kernel and add semantics kernel = reduce_to_section(parsed_code, isa) machine_model = MachineModel(arch=arch) - semantics = SemanticsAppender(machine_model) + semantics = ArchSemantics(machine_model) semantics.add_semantics(kernel) # Create DiGrahps diff --git a/osaca/semantics/__init__.py b/osaca/semantics/__init__.py index 167cef1..e5612a0 100644 --- a/osaca/semantics/__init__.py +++ b/osaca/semantics/__init__.py @@ -3,9 +3,17 @@ Tools for semantic analysis of parser result. Only the classes below will be exported, so please add new semantic tools to __all__. """ +from .isa_semantics import ISASemantics, INSTR_FLAGS +from .arch_semantics import ArchSemantics from .hw_model import MachineModel from .kernel_dg import KernelDG from .marker_utils import reduce_to_section -from .semantics_appender import SemanticsAppender, INSTR_FLAGS -__all__ = ['MachineModel', 'KernelDG', 'reduce_to_section', 'SemanticsAppender', 'INSTR_FLAGS'] +__all__ = [ + 'MachineModel', + 'KernelDG', + 'reduce_to_section', + 'ArchSemantics', + 'ISASemantics', + 'INSTR_FLAGS', +] diff --git a/osaca/semantics/semantics_appender.py b/osaca/semantics/arch_semantics.py similarity index 71% rename from osaca/semantics/semantics_appender.py rename to osaca/semantics/arch_semantics.py index 4b8daa9..ba44ab6 100755 --- a/osaca/semantics/semantics_appender.py +++ b/osaca/semantics/arch_semantics.py @@ -3,35 +3,16 @@ import warnings from functools import reduce -from osaca import utils -from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT -from osaca.semantics import MachineModel +from osaca.semantics.isa_semantics import ISASemantics +from osaca.semantics.hw_model import MachineModel +from osaca.semantics.instr_flags import INSTR_FLAGS -class INSTR_FLAGS: - """ - Flags used for unknown or special instructions - """ - - LD = 'is_load_instruction' - TP_UNKWN = 'tp_unknown' - LT_UNKWN = 'lt_unknown' - NOT_BOUND = 'not_bound' - HIDDEN_LD = 'hidden_load' - HAS_LD = 'performs_load' - HAS_ST = 'performs_store' - - -class SemanticsAppender(object): +class ArchSemantics(ISASemantics): def __init__(self, machine_model: MachineModel, path_to_yaml=None): + super().__init__(machine_model.get_ISA().lower(), path_to_yaml=path_to_yaml) self._machine_model = machine_model self._isa = machine_model.get_ISA().lower() - path = utils.find_file('isa/' + self._isa + '.yml') - self._isa_model = MachineModel(path_to_yaml=path) - if self._isa == 'x86': - self._parser = ParserX86ATT() - elif self._isa == 'aarch64': - self._parser = ParserAArch64v81() # SUMMARY FUNCTION def add_semantics(self, kernel): @@ -234,53 +215,6 @@ class SemanticsAppender(object): register = {'register': {'prefix': reg_type, 'name': reg_id}} return register - # get ;parser result and assign operands to - # - source - # - destination - # - source/destination - def assign_src_dst(self, instruction_form): - # if the instruction form doesn't have operands, there's nothing to do - if instruction_form['operands'] is None: - return - # check if instruction form is in ISA yaml, otherwise apply standard operand assignment - # (one dest, others source) - isa_data = self._isa_model.get_instruction( - instruction_form['instruction'], instruction_form['operands'] - ) - operands = instruction_form['operands'] - op_dict = {} - if isa_data is None: - # no irregular operand structure, apply default - op_dict['source'] = self._get_regular_source_operands(instruction_form) - op_dict['destination'] = self._get_regular_destination_operands(instruction_form) - op_dict['src_dst'] = [] - else: - # load src/dst structure from isa_data - op_dict['source'] = [] - op_dict['destination'] = [] - op_dict['src_dst'] = [] - for i, op in enumerate(isa_data['operands']): - if op['source'] and op['destination']: - op_dict['src_dst'].append(operands[i]) - continue - if op['source']: - op_dict['source'].append(operands[i]) - continue - if op['destination']: - op_dict['destination'].append(operands[i]) - continue - # store operand list in dict and reassign operand key/value pair - op_dict['operand_list'] = operands - instruction_form['operands'] = AttrDict.convert_dict(op_dict) - # assign LD/ST flags - instruction_form['flags'] = ( - instruction_form['flags'] if 'flags' in instruction_form else [] - ) - if self._has_load(instruction_form): - instruction_form['flags'] += [INSTR_FLAGS.HAS_LD] - if self._has_store(instruction_form): - instruction_form['flags'] += [INSTR_FLAGS.HAS_ST] - def _nullify_data_ports(self, port_pressure): data_ports = self._machine_model.get_data_ports() for port in data_ports: @@ -288,42 +222,6 @@ class SemanticsAppender(object): port_pressure[index] = 0.0 return port_pressure - def _has_load(self, instruction_form): - for operand in ( - instruction_form['operands']['source'] + instruction_form['operands']['src_dst'] - ): - if 'memory' in operand: - return True - return False - - def _has_store(self, instruction_form): - for operand in ( - instruction_form['operands']['destination'] + instruction_form['operands']['src_dst'] - ): - if 'memory' in operand: - return True - return False - - def _get_regular_source_operands(self, instruction_form): - if self._isa == 'x86': - # return all but last operand - return [op - or op in instruction_form['operands'][0:len(instruction_form['operands']) - 1]] - elif self._isa == 'aarch64': - return [op for op in instruction_form['operands'][1:len(instruction_form['operands'])]] - else: - raise ValueError("Unsupported ISA {}.".format(self._isa)) - - def _get_regular_destination_operands(self, instruction_form): - if self._isa == 'x86': - # return last operand - return instruction_form['operands'][-1:] - if self._isa == 'aarch64': - # return first operand - return instruction_form['operands'][:1] - else: - raise ValueError("Unsupported ISA {}.".format(self._isa)) - @staticmethod def get_throughput_sum(kernel): tp_sum = reduce( diff --git a/osaca/semantics/isa_semantics.py b/osaca/semantics/isa_semantics.py new file mode 100755 index 0000000..71b8e7a --- /dev/null +++ b/osaca/semantics/isa_semantics.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +from osaca import utils +from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT +from osaca.semantics.hw_model import MachineModel + + +class INSTR_FLAGS: + """ + Flags used for unknown or special instructions + """ + + LD = 'is_load_instruction' + TP_UNKWN = 'tp_unknown' + LT_UNKWN = 'lt_unknown' + NOT_BOUND = 'not_bound' + HIDDEN_LD = 'hidden_load' + HAS_LD = 'performs_load' + HAS_ST = 'performs_store' + + +class ISASemantics(object): + def __init__(self, isa, path_to_yaml=None): + self._isa = isa.lower() + path = utils.find_file('isa/' + self._isa + '.yml') if not path_to_yaml else path_to_yaml + self._isa_model = MachineModel(path_to_yaml=path) + if self._isa == 'x86': + self._parser = ParserX86ATT() + elif self._isa == 'aarch64': + self._parser = ParserAArch64v81() + + # get ;parser result and assign operands to + # - source + # - destination + # - source/destination + def assign_src_dst(self, instruction_form): + # if the instruction form doesn't have operands, there's nothing to do + if instruction_form['operands'] is None: + return + # check if instruction form is in ISA yaml, otherwise apply standard operand assignment + # (one dest, others source) + isa_data = self._isa_model.get_instruction( + instruction_form['instruction'], instruction_form['operands'] + ) + operands = instruction_form['operands'] + op_dict = {} + if isa_data is None: + # no irregular operand structure, apply default + op_dict['source'] = self._get_regular_source_operands(instruction_form) + op_dict['destination'] = self._get_regular_destination_operands(instruction_form) + op_dict['src_dst'] = [] + else: + # load src/dst structure from isa_data + op_dict['source'] = [] + op_dict['destination'] = [] + op_dict['src_dst'] = [] + for i, op in enumerate(isa_data['operands']): + if op['source'] and op['destination']: + op_dict['src_dst'].append(operands[i]) + continue + if op['source']: + op_dict['source'].append(operands[i]) + continue + if op['destination']: + op_dict['destination'].append(operands[i]) + continue + # store operand list in dict and reassign operand key/value pair + op_dict['operand_list'] = operands + instruction_form['operands'] = AttrDict.convert_dict(op_dict) + # assign LD/ST flags + instruction_form['flags'] = ( + instruction_form['flags'] if 'flags' in instruction_form else [] + ) + if self._has_load(instruction_form): + instruction_form['flags'] += [INSTR_FLAGS.HAS_LD] + if self._has_store(instruction_form): + instruction_form['flags'] += [INSTR_FLAGS.HAS_ST] + + def _has_load(self, instruction_form): + for operand in ( + instruction_form['operands']['source'] + instruction_form['operands']['src_dst'] + ): + if 'memory' in operand: + return True + return False + + def _has_store(self, instruction_form): + for operand in ( + instruction_form['operands']['destination'] + instruction_form['operands']['src_dst'] + ): + if 'memory' in operand: + return True + return False + + def _get_regular_source_operands(self, instruction_form): + if self._isa == 'x86': + # return all but last operand + return [ + op for op in instruction_form['operands'][0:len(instruction_form['operands']) - 1] + ] + elif self._isa == 'aarch64': + return [ + op for op in instruction_form['operands'][1:len(instruction_form['operands'])] + ] + else: + raise ValueError("Unsupported ISA {}.".format(self._isa)) + + def _get_regular_destination_operands(self, instruction_form): + if self._isa == 'x86': + # return last operand + return instruction_form['operands'][-1:] + if self._isa == 'aarch64': + # return first operand + return instruction_form['operands'][:1] + else: + raise ValueError("Unsupported ISA {}.".format(self._isa)) diff --git a/osaca/semantics/marker_utils.py b/osaca/semantics/marker_utils.py index ecec7c0..9e00233 100755 --- a/osaca/semantics/marker_utils.py +++ b/osaca/semantics/marker_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from collections import OrderedDict -from osaca.parser import get_parser, ParserAArch64v81, ParserX86ATT +from osaca.parser import ParserAArch64v81, ParserX86ATT, get_parser def reduce_to_section(kernel, isa): @@ -28,7 +28,9 @@ def find_marked_kernel_AArch64(lines): def find_marked_kernel_x86ATT(lines): nop_bytes = ['100', '103', '144'] - return find_marked_section(lines, ParserX86ATT(), ['mov', 'movl'], 'ebx', [111, 222], nop_bytes) + return find_marked_section( + lines, ParserX86ATT(), ['mov', 'movl'], 'ebx', [111, 222], nop_bytes + ) def get_marker(isa): @@ -38,20 +40,24 @@ def get_marker(isa): 'movl $111, %ebx # OSACA START MARKER\n' '.byte 100 # OSACA START MARKER\n' '.byte 103 # OSACA START MARKER\n' - '.byte 144 # OSACA START MARKER\n') + '.byte 144 # OSACA START MARKER\n' + ) end_marker_raw = ( 'movl $222, %ebx # OSACA END MARKER\n' '.byte 100 # OSACA END MARKER\n' '.byte 103 # OSACA END MARKER\n' - '.byte 144 # OSACA END MARKER\n') + '.byte 144 # OSACA END MARKER\n' + ) elif isa == 'AArch64': start_marker_raw = ( 'mov x1, #111 // OSACA START MARKER\n' - '.byte 213,3,32,31 // OSACA START MARKER\n') + '.byte 213,3,32,31 // OSACA START MARKER\n' + ) # After loop end_marker_raw = ( 'mov x1, #222 // OSACA END MARKER\n' - '.byte 213,3,32,31 // OSACA END MARKER\n') + '.byte 213,3,32,31 // OSACA END MARKER\n' + ) parser = get_parser(isa) start_marker = parser.parse_file(start_marker_raw) @@ -127,7 +133,7 @@ def find_jump_labels(lines): for i, line in enumerate(lines): if line['label'] is not None: # When a new label is found, add to blocks dict - labels[line['label']] = (i, ) + labels[line['label']] = (i,) # End previous block at previous line if current_label is not None: labels[current_label] = (labels[current_label][0], i) @@ -138,12 +144,17 @@ def find_jump_labels(lines): continue # Set to last line if no end was for last label found if current_label is not None and len(labels[current_label]) == 1: - labels[current_label] = (labels[current_label][0], i+1) + labels[current_label] = (labels[current_label][0], i + 1) # 2. Identify and remove labels which contain only dot-instructions (e.g., .text) for label in list(labels): - if all([l['instruction'].startswith('.') - for l in lines[labels[label][0]:labels[label][1]] if l['instruction'] is not None]): + if all( + [ + l['instruction'].startswith('.') + for l in lines[labels[label][0]:labels[label][1]] + if l['instruction'] is not None + ] + ): del labels[label] return OrderedDict([(l, lines[v[0]]) for l, v in labels.items()]) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 7d6dca8..d4863f8 100755 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -8,9 +8,7 @@ import unittest from osaca.frontend import Frontend from osaca.parser import ParserAArch64v81, ParserX86ATT -from osaca.semantics.hw_model import MachineModel -from osaca.semantics.kernel_dg import KernelDG -from osaca.semantics.semantics_appender import SemanticsAppender +from osaca.semantics import ArchSemantics, KernelDG, MachineModel class TestFrontend(unittest.TestCase): @@ -37,10 +35,10 @@ class TestFrontend(unittest.TestCase): self.machine_model_tx2 = MachineModel( path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') ) - self.semantics_csx = SemanticsAppender( + self.semantics_csx = ArchSemantics( self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') ) - self.semantics_tx2 = SemanticsAppender( + self.semantics_tx2 = ArchSemantics( self.machine_model_tx2, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'), ) diff --git a/tests/test_semantics.py b/tests/test_semantics.py index 324c482..f5c5a8b 100755 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -11,7 +11,7 @@ import networkx as nx from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel, - SemanticsAppender) + ArchSemantics) class TestSemanticTools(unittest.TestCase): @@ -43,10 +43,10 @@ class TestSemanticTools(unittest.TestCase): self.machine_model_tx2 = MachineModel( path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') ) - self.semantics_csx = SemanticsAppender( + self.semantics_csx = ArchSemantics( self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') ) - self.semantics_tx2 = SemanticsAppender( + self.semantics_tx2 = ArchSemantics( self.machine_model_tx2, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'), ) @@ -66,7 +66,7 @@ class TestSemanticTools(unittest.TestCase): def test_creation_by_name(self): try: tmp_mm = MachineModel(arch='CSX') - SemanticsAppender(tmp_mm) + ArchSemantics(tmp_mm) except ValueError: self.fail() @@ -158,7 +158,7 @@ class TestSemanticTools(unittest.TestCase): path_to_yaml=self._find_file('hidden_load_machine_model.yml') ) self.assertTrue(machine_model_hld.has_hidden_loads()) - semantics_hld = SemanticsAppender(machine_model_hld) + semantics_hld = ArchSemantics(machine_model_hld) kernel_hld = self.parser_x86.parse_file(self.code_x86) kernel_hld_2 = self.parser_x86.parse_file(self.code_x86) kernel_hld_2 = self.parser_x86.parse_file(self.code_x86)[-3:]