Lerking/OSACA
1
0
Fork 0
mirror of https://github.com/RRZE-HPC/OSACA.git synced 2025-12-16 00:50:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

separated SemanticsAppender into ISA and Arch semantics

Browse Source
This commit is contained in:
JanLJL
2019-10-29 09:09:52 +01:00
parent 1c8067545d
commit 0f5d3a0370
9 changed files with 167 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
osaca/api/kerncraft_interface.py
View File

@@ -7,7 +7,7 @@ from io import StringIO
from osaca.frontend import Frontend
from osaca.parser import ParserAArch64v81, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel,
SemanticsAppender, reduce_to_section)
ArchSemantics, reduce_to_section)
# Stolen from https://stackoverflow.com/a/16571630
@@ -26,7 +26,7 @@ class Capturing(list):
class KerncraftAPI(object):
def __init__(self, arch, code):
self.machine_model = MachineModel(arch=arch)
self.semantics = SemanticsAppender(self.machine_model)
self.semantics = ArchSemantics(self.machine_model)
isa = self.machine_model.get_ISA().lower()
if isa == 'aarch64':
self.parser = ParserAArch64v81()

6
osaca/frontend.py
View File

@@ -6,7 +6,7 @@ from datetime import datetime as dt
from ruamel import yaml
from osaca import utils
from osaca.semantics import INSTR_FLAGS, KernelDG, SemanticsAppender
from osaca.semantics import INSTR_FLAGS, KernelDG, ArchSemantics
class Frontend(object):
@@ -56,7 +56,7 @@ class Frontend(object):
continue
print(line)
print()
tp_sum = SemanticsAppender.get_throughput_sum(kernel)
tp_sum = ArchSemantics.get_throughput_sum(kernel)
print(lineno_filler + self._get_port_pressure(tp_sum, port_len, ' '))
def print_latency_analysis(self, cp_kernel, separator='|'):
@@ -165,7 +165,7 @@ class Frontend(object):
print(line)
print()
# lcd_sum already calculated before
tp_sum = SemanticsAppender.get_throughput_sum(kernel)
tp_sum = ArchSemantics.get_throughput_sum(kernel)
cp_sum = sum([x['latency_cp'] for x in cp_kernel])
print(
lineno_filler

4
osaca/osaca.py
View File

@@ -10,7 +10,7 @@ from subprocess import call
from osaca.db_interface import sanity_check, import_benchmark_output
from osaca.frontend import Frontend
from osaca.parser import BaseParser, ParserAArch64v81, ParserX86ATT
from osaca.semantics import (KernelDG, MachineModel, SemanticsAppender,
from osaca.semantics import (KernelDG, MachineModel, ArchSemantics,
reduce_to_section)
MODULE_DATA_DIR = os.path.join(
@@ -181,7 +181,7 @@ def inspect(args):
# Reduce to marked kernel and add semantics
kernel = reduce_to_section(parsed_code, isa)
machine_model = MachineModel(arch=arch)
semantics = SemanticsAppender(machine_model)
semantics = ArchSemantics(machine_model)
semantics.add_semantics(kernel)
# Create DiGrahps

12
osaca/semantics/__init__.py
View File

@@ -3,9 +3,17 @@ Tools for semantic analysis of parser result.
Only the classes below will be exported, so please add new semantic tools to __all__.
"""
from .isa_semantics import ISASemantics, INSTR_FLAGS
from .arch_semantics import ArchSemantics
from .hw_model import MachineModel
from .kernel_dg import KernelDG
from .marker_utils import reduce_to_section
from .semantics_appender import SemanticsAppender, INSTR_FLAGS
__all__ = ['MachineModel', 'KernelDG', 'reduce_to_section', 'SemanticsAppender', 'INSTR_FLAGS']
__all__ = [
'MachineModel',
'KernelDG',
'reduce_to_section',
'ArchSemantics',
'ISASemantics',
'INSTR_FLAGS',
]

112
osaca/semantics/semantics_appender.py → osaca/semantics/arch_semantics.py
View File

@@ -3,35 +3,16 @@
import warnings
from functools import reduce
from osaca import utils
from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT
from osaca.semantics import MachineModel
from osaca.semantics.isa_semantics import ISASemantics
from osaca.semantics.hw_model import MachineModel
from osaca.semantics.instr_flags import INSTR_FLAGS
class INSTR_FLAGS:
"""
Flags used for unknown or special instructions
"""
LD = 'is_load_instruction'
TP_UNKWN = 'tp_unknown'
LT_UNKWN = 'lt_unknown'
NOT_BOUND = 'not_bound'
HIDDEN_LD = 'hidden_load'
HAS_LD = 'performs_load'
HAS_ST = 'performs_store'
class SemanticsAppender(object):
class ArchSemantics(ISASemantics):
def __init__(self, machine_model: MachineModel, path_to_yaml=None):
super().__init__(machine_model.get_ISA().lower(), path_to_yaml=path_to_yaml)
self._machine_model = machine_model
self._isa = machine_model.get_ISA().lower()
path = utils.find_file('isa/' + self._isa + '.yml')
self._isa_model = MachineModel(path_to_yaml=path)
if self._isa == 'x86':
self._parser = ParserX86ATT()
elif self._isa == 'aarch64':
self._parser = ParserAArch64v81()
# SUMMARY FUNCTION
def add_semantics(self, kernel):
@@ -234,53 +215,6 @@ class SemanticsAppender(object):
register = {'register': {'prefix': reg_type, 'name': reg_id}}
return register
# get ;parser result and assign operands to
# - source
# - destination
# - source/destination
def assign_src_dst(self, instruction_form):
# if the instruction form doesn't have operands, there's nothing to do
if instruction_form['operands'] is None:
return
# check if instruction form is in ISA yaml, otherwise apply standard operand assignment
# (one dest, others source)
isa_data = self._isa_model.get_instruction(
instruction_form['instruction'], instruction_form['operands']
)
operands = instruction_form['operands']
op_dict = {}
if isa_data is None:
# no irregular operand structure, apply default
op_dict['source'] = self._get_regular_source_operands(instruction_form)
op_dict['destination'] = self._get_regular_destination_operands(instruction_form)
op_dict['src_dst'] = []
else:
# load src/dst structure from isa_data
op_dict['source'] = []
op_dict['destination'] = []
op_dict['src_dst'] = []
for i, op in enumerate(isa_data['operands']):
if op['source'] and op['destination']:
op_dict['src_dst'].append(operands[i])
continue
if op['source']:
op_dict['source'].append(operands[i])
continue
if op['destination']:
op_dict['destination'].append(operands[i])
continue
# store operand list in dict and reassign operand key/value pair
op_dict['operand_list'] = operands
instruction_form['operands'] = AttrDict.convert_dict(op_dict)
# assign LD/ST flags
instruction_form['flags'] = (
instruction_form['flags'] if 'flags' in instruction_form else []
)
if self._has_load(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_LD]
if self._has_store(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_ST]
def _nullify_data_ports(self, port_pressure):
data_ports = self._machine_model.get_data_ports()
for port in data_ports:
@@ -288,42 +222,6 @@ class SemanticsAppender(object):
port_pressure[index] = 0.0
return port_pressure
def _has_load(self, instruction_form):
for operand in (
instruction_form['operands']['source'] + instruction_form['operands']['src_dst']
):
if 'memory' in operand:
return True
return False
def _has_store(self, instruction_form):
for operand in (
instruction_form['operands']['destination'] + instruction_form['operands']['src_dst']
):
if 'memory' in operand:
return True
return False
def _get_regular_source_operands(self, instruction_form):
if self._isa == 'x86':
# return all but last operand
return [op
or op in instruction_form['operands'][0:len(instruction_form['operands']) - 1]]
elif self._isa == 'aarch64':
return [op for op in instruction_form['operands'][1:len(instruction_form['operands'])]]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))
def _get_regular_destination_operands(self, instruction_form):
if self._isa == 'x86':
# return last operand
return instruction_form['operands'][-1:]
if self._isa == 'aarch64':
# return first operand
return instruction_form['operands'][:1]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))
@staticmethod
def get_throughput_sum(kernel):
tp_sum = reduce(

116
osaca/semantics/isa_semantics.py Executable file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env python3
from osaca import utils
from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT
from osaca.semantics.hw_model import MachineModel
class INSTR_FLAGS:
"""
Flags used for unknown or special instructions
"""
LD = 'is_load_instruction'
TP_UNKWN = 'tp_unknown'
LT_UNKWN = 'lt_unknown'
NOT_BOUND = 'not_bound'
HIDDEN_LD = 'hidden_load'
HAS_LD = 'performs_load'
HAS_ST = 'performs_store'
class ISASemantics(object):
def __init__(self, isa, path_to_yaml=None):
self._isa = isa.lower()
path = utils.find_file('isa/' + self._isa + '.yml') if not path_to_yaml else path_to_yaml
self._isa_model = MachineModel(path_to_yaml=path)
if self._isa == 'x86':
self._parser = ParserX86ATT()
elif self._isa == 'aarch64':
self._parser = ParserAArch64v81()
# get ;parser result and assign operands to
# - source
# - destination
# - source/destination
def assign_src_dst(self, instruction_form):
# if the instruction form doesn't have operands, there's nothing to do
if instruction_form['operands'] is None:
return
# check if instruction form is in ISA yaml, otherwise apply standard operand assignment
# (one dest, others source)
isa_data = self._isa_model.get_instruction(
instruction_form['instruction'], instruction_form['operands']
)
operands = instruction_form['operands']
op_dict = {}
if isa_data is None:
# no irregular operand structure, apply default
op_dict['source'] = self._get_regular_source_operands(instruction_form)
op_dict['destination'] = self._get_regular_destination_operands(instruction_form)
op_dict['src_dst'] = []
else:
# load src/dst structure from isa_data
op_dict['source'] = []
op_dict['destination'] = []
op_dict['src_dst'] = []
for i, op in enumerate(isa_data['operands']):
if op['source'] and op['destination']:
op_dict['src_dst'].append(operands[i])
continue
if op['source']:
op_dict['source'].append(operands[i])
continue
if op['destination']:
op_dict['destination'].append(operands[i])
continue
# store operand list in dict and reassign operand key/value pair
op_dict['operand_list'] = operands
instruction_form['operands'] = AttrDict.convert_dict(op_dict)
# assign LD/ST flags
instruction_form['flags'] = (
instruction_form['flags'] if 'flags' in instruction_form else []
)
if self._has_load(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_LD]
if self._has_store(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_ST]
def _has_load(self, instruction_form):
for operand in (
instruction_form['operands']['source'] + instruction_form['operands']['src_dst']
):
if 'memory' in operand:
return True
return False
def _has_store(self, instruction_form):
for operand in (
instruction_form['operands']['destination'] + instruction_form['operands']['src_dst']
):
if 'memory' in operand:
return True
return False
def _get_regular_source_operands(self, instruction_form):
if self._isa == 'x86':
# return all but last operand
return [
op for op in instruction_form['operands'][0:len(instruction_form['operands']) - 1]
]
elif self._isa == 'aarch64':
return [
op for op in instruction_form['operands'][1:len(instruction_form['operands'])]
]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))
def _get_regular_destination_operands(self, instruction_form):
if self._isa == 'x86':
# return last operand
return instruction_form['operands'][-1:]
if self._isa == 'aarch64':
# return first operand
return instruction_form['operands'][:1]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))

31
osaca/semantics/marker_utils.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
from collections import OrderedDict
from osaca.parser import get_parser, ParserAArch64v81, ParserX86ATT
from osaca.parser import ParserAArch64v81, ParserX86ATT, get_parser
def reduce_to_section(kernel, isa):
@@ -28,7 +28,9 @@ def find_marked_kernel_AArch64(lines):
def find_marked_kernel_x86ATT(lines):
nop_bytes = ['100', '103', '144']
return find_marked_section(lines, ParserX86ATT(), ['mov', 'movl'], 'ebx', [111, 222], nop_bytes)
return find_marked_section(
lines, ParserX86ATT(), ['mov', 'movl'], 'ebx', [111, 222], nop_bytes
)
def get_marker(isa):
@@ -38,20 +40,24 @@ def get_marker(isa):
'movl $111, %ebx # OSACA START MARKER\n'
'.byte 100 # OSACA START MARKER\n'
'.byte 103 # OSACA START MARKER\n'
'.byte 144 # OSACA START MARKER\n')
'.byte 144 # OSACA START MARKER\n'
)
end_marker_raw = (
'movl $222, %ebx # OSACA END MARKER\n'
'.byte 100 # OSACA END MARKER\n'
'.byte 103 # OSACA END MARKER\n'
'.byte 144 # OSACA END MARKER\n')
'.byte 144 # OSACA END MARKER\n'
)
elif isa == 'AArch64':
start_marker_raw = (
'mov x1, #111 // OSACA START MARKER\n'
'.byte 213,3,32,31 // OSACA START MARKER\n')
'.byte 213,3,32,31 // OSACA START MARKER\n'
)
# After loop
end_marker_raw = (
'mov x1, #222 // OSACA END MARKER\n'
'.byte 213,3,32,31 // OSACA END MARKER\n')
'.byte 213,3,32,31 // OSACA END MARKER\n'
)
parser = get_parser(isa)
start_marker = parser.parse_file(start_marker_raw)
@@ -127,7 +133,7 @@ def find_jump_labels(lines):
for i, line in enumerate(lines):
if line['label'] is not None:
# When a new label is found, add to blocks dict
labels[line['label']] = (i, )
labels[line['label']] = (i,)
# End previous block at previous line
if current_label is not None:
labels[current_label] = (labels[current_label][0], i)
@@ -138,12 +144,17 @@ def find_jump_labels(lines):
continue
# Set to last line if no end was for last label found
if current_label is not None and len(labels[current_label]) == 1:
labels[current_label] = (labels[current_label][0], i+1)
labels[current_label] = (labels[current_label][0], i + 1)
# 2. Identify and remove labels which contain only dot-instructions (e.g., .text)
for label in list(labels):
if all([l['instruction'].startswith('.')
for l in lines[labels[label][0]:labels[label][1]] if l['instruction'] is not None]):
if all(
[
l['instruction'].startswith('.')
for l in lines[labels[label][0]:labels[label][1]]
if l['instruction'] is not None
]
):
del labels[label]
return OrderedDict([(l, lines[v[0]]) for l, v in labels.items()])

8
tests/test_frontend.py
View File

@@ -8,9 +8,7 @@ import unittest
from osaca.frontend import Frontend
from osaca.parser import ParserAArch64v81, ParserX86ATT
from osaca.semantics.hw_model import MachineModel
from osaca.semantics.kernel_dg import KernelDG
from osaca.semantics.semantics_appender import SemanticsAppender
from osaca.semantics import ArchSemantics, KernelDG, MachineModel
class TestFrontend(unittest.TestCase):
@@ -37,10 +35,10 @@ class TestFrontend(unittest.TestCase):
self.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml')
)
self.semantics_csx = SemanticsAppender(
self.semantics_csx = ArchSemantics(
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml')
)
self.semantics_tx2 = SemanticsAppender(
self.semantics_tx2 = ArchSemantics(
self.machine_model_tx2,
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'),
)

10
tests/test_semantics.py
View File

@@ -11,7 +11,7 @@ import networkx as nx
from osaca.parser import AttrDict, ParserAArch64v81, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel,
SemanticsAppender)
ArchSemantics)
class TestSemanticTools(unittest.TestCase):
@@ -43,10 +43,10 @@ class TestSemanticTools(unittest.TestCase):
self.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml')
)
self.semantics_csx = SemanticsAppender(
self.semantics_csx = ArchSemantics(
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml')
)
self.semantics_tx2 = SemanticsAppender(
self.semantics_tx2 = ArchSemantics(
self.machine_model_tx2,
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'),
)
@@ -66,7 +66,7 @@ class TestSemanticTools(unittest.TestCase):
def test_creation_by_name(self):
try:
tmp_mm = MachineModel(arch='CSX')
SemanticsAppender(tmp_mm)
ArchSemantics(tmp_mm)
except ValueError:
self.fail()
@@ -158,7 +158,7 @@ class TestSemanticTools(unittest.TestCase):
path_to_yaml=self._find_file('hidden_load_machine_model.yml')
)
self.assertTrue(machine_model_hld.has_hidden_loads())
semantics_hld = SemanticsAppender(machine_model_hld)
semantics_hld = ArchSemantics(machine_model_hld)
kernel_hld = self.parser_x86.parse_file(self.code_x86)
kernel_hld_2 = self.parser_x86.parse_file(self.code_x86)
kernel_hld_2 = self.parser_x86.parse_file(self.code_x86)[-3:]