From 6204c90934c0e62aed98862ae77368b20a64cbfb Mon Sep 17 00:00:00 2001 From: Julian Hammer Date: Thu, 11 Mar 2021 12:02:45 +0100 Subject: [PATCH] migrate code style to Black --- docs/conf.py | 38 +- docs/version_from_src.py | 4 +- osaca/__init__.py | 4 +- osaca/api/__init__.py | 2 +- osaca/api/kerncraft_interface.py | 19 +- osaca/data/_build_cache.py | 23 +- osaca/data/generate_mov_entries.py | 1020 ++++++++++++++-------------- osaca/data/model_importer.py | 219 +++--- osaca/db_interface.py | 409 +++++------ osaca/frontend.py | 294 ++++---- osaca/osaca.py | 206 +++--- osaca/parser/__init__.py | 7 +- osaca/parser/base_parser.py | 35 +- osaca/parser/parser_AArch64.py | 368 +++++----- osaca/parser/parser_x86att.py | 277 ++++---- osaca/semantics/__init__.py | 18 +- osaca/semantics/arch_semantics.py | 202 +++--- osaca/semantics/hw_model.py | 561 +++++++-------- osaca/semantics/isa_semantics.py | 168 ++--- osaca/semantics/kernel_dg.py | 184 +++-- osaca/semantics/marker_utils.py | 92 +-- osaca/utils.py | 4 +- setup.py | 91 +-- tests/all_tests.py | 20 +- tests/test_base_parser.py | 32 +- tests/test_cli.py | 153 +++-- tests/test_db_interface.py | 108 +-- tests/test_frontend.py | 32 +- tests/test_kerncraftAPI.py | 50 +- tests/test_marker_utils.py | 405 ++++++----- tests/test_parser_AArch64.py | 445 ++++++------ tests/test_parser_x86att.py | 268 ++++---- tests/test_semantics.py | 247 ++++--- 33 files changed, 3045 insertions(+), 2960 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 68fde48..e1cb993 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,15 +8,15 @@ import os import sys -sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath(".")) from version_from_src import get_version # -- Project information ----------------------------------------------------- -project = 'OSACA' -copyright = '2020, Jan Laukemann' -author = 'Jan Laukemann' -html_logo = 'img/osaca-logo.png' +project = "OSACA" +copyright = "2020, Jan Laukemann" +author = "Jan Laukemann" +html_logo = "img/osaca-logo.png" # The full version, including alpha/beta/rc tags version = get_version() @@ -28,21 +28,21 @@ release = get_version() # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.mathjax', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.viewcode', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", ] add_module_names = False -source_suffix = '.rst' -master_doc = 'index' +source_suffix = ".rst" +master_doc = "index" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -55,13 +55,13 @@ exclude_patterns = [] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # e.g., 'alabaster', 'sphinx_rtd_theme' -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] -htmlhelp_basename = 'osaca_doc' -html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']} +htmlhelp_basename = "osaca_doc" +html_sidebars = {"**": ["globaltoc.html", "relations.html", "sourcelink.html", "searchbox.html"]} -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" diff --git a/docs/version_from_src.py b/docs/version_from_src.py index a2aa1a4..97a4cda 100644 --- a/docs/version_from_src.py +++ b/docs/version_from_src.py @@ -19,7 +19,7 @@ def __find_version(*file_paths): version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) if version_match: return version_match.group(1) - raise RuntimeError('Unable to find version string.') + raise RuntimeError("Unable to find version string.") def get_version(): @@ -28,4 +28,4 @@ def get_version(): :returns: str -- the version string. """ - return __find_version('../osaca/__init__.py') + return __find_version("../osaca/__init__.py") diff --git a/osaca/__init__.py b/osaca/__init__.py index 8dca503..beca6bc 100644 --- a/osaca/__init__.py +++ b/osaca/__init__.py @@ -1,6 +1,6 @@ """Open Source Architecture Code Analyzer""" -name = 'osaca' -__version__ = '0.3.14' +name = "osaca" +__version__ = "0.3.14" # To trigger travis deployment to pypi, do the following: # 1. Increment __version___ diff --git a/osaca/api/__init__.py b/osaca/api/__init__.py index ec3d818..3d7d60a 100644 --- a/osaca/api/__init__.py +++ b/osaca/api/__init__.py @@ -5,4 +5,4 @@ Only the classes below will be exported, so please add new semantic tools to __a """ from .kerncraft_interface import KerncraftAPI -__all__ = ['KerncraftAPI'] +__all__ = ["KerncraftAPI"] diff --git a/osaca/api/kerncraft_interface.py b/osaca/api/kerncraft_interface.py index 2d448e1..451e50e 100755 --- a/osaca/api/kerncraft_interface.py +++ b/osaca/api/kerncraft_interface.py @@ -6,8 +6,7 @@ from io import StringIO from osaca.frontend import Frontend from osaca.parser import ParserAArch64, ParserX86ATT -from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel, - ArchSemantics, reduce_to_section) +from osaca.semantics import INSTR_FLAGS, KernelDG, MachineModel, ArchSemantics, reduce_to_section # Stolen from https://stackoverflow.com/a/16571630 @@ -19,7 +18,7 @@ class Capturing(list): def __exit__(self, *args): self.extend(self._stringio.getvalue().splitlines()) - del self._stringio # free up some memory + del self._stringio # free up some memory sys.stdout = self._stdout @@ -28,9 +27,9 @@ class KerncraftAPI(object): self.machine_model = MachineModel(arch=arch) self.semantics = ArchSemantics(self.machine_model) isa = self.machine_model.get_ISA().lower() - if isa == 'aarch64': + if isa == "aarch64": self.parser = ParserAArch64() - elif isa == 'x86': + elif isa == "x86": self.parser = ParserX86ATT() parsed_code = self.parser.parse_file(code) @@ -46,15 +45,15 @@ class KerncraftAPI(object): unmatched_counter = 0 for instruction in self.kernel: if ( - INSTR_FLAGS.TP_UNKWN in instruction['flags'] - and INSTR_FLAGS.LT_UNKWN in instruction['flags'] + INSTR_FLAGS.TP_UNKWN in instruction["flags"] + and INSTR_FLAGS.LT_UNKWN in instruction["flags"] ): unmatched_counter += 1 return unmatched_counter / len(self.kernel) def get_port_occupation_cycles(self): throughput_values = self.semantics.get_throughput_sum(self.kernel) - port_names = self.machine_model['ports'] + port_names = self.machine_model["ports"] return collections.OrderedDict(list(zip(port_names, throughput_values))) def get_total_throughput(self): @@ -66,13 +65,13 @@ class KerncraftAPI(object): def get_cp(self): kernel_graph = KernelDG(self.kernel, self.parser, self.machine_model) kernel_cp = kernel_graph.get_critical_path() - return sum([x['latency_cp'] for x in kernel_cp]) + return sum([x["latency_cp"] for x in kernel_cp]) def get_lcd(self): kernel_graph = KernelDG(self.kernel, self.parser, self.machine_model) lcd_dict = kernel_graph.get_loopcarried_dependencies() lcd = 0.0 for dep in lcd_dict: - lcd_tmp = sum([x['latency_lcd'] for x in lcd_dict[dep]['dependencies']]) + lcd_tmp = sum([x["latency_lcd"] for x in lcd_dict[dep]["dependencies"]]) lcd = lcd_tmp if lcd_tmp > lcd else lcd return lcd diff --git a/osaca/data/_build_cache.py b/osaca/data/_build_cache.py index 76b650c..74803cb 100755 --- a/osaca/data/_build_cache.py +++ b/osaca/data/_build_cache.py @@ -2,30 +2,33 @@ from glob import glob import os.path import sys -sys.path[0:0] = ['../..'] + +sys.path[0:0] = ["../.."] failed = False try: from osaca.semantics.hw_model import MachineModel except ModuleNotFoundError: - print("Unable to import MachineModel, probably some dependency is not yet installed. SKIPPING. " - "First run of OSACA may take a while to build caches, subsequent runs will be as fast as " - "ever.") + print( + "Unable to import MachineModel, probably some dependency is not yet installed. SKIPPING. " + "First run of OSACA may take a while to build caches, subsequent runs will be as fast as " + "ever." + ) sys.exit() -print('Building cache: ', end='') +print("Building cache: ", end="") sys.stdout.flush() # Iterating architectures -for f in glob(os.path.join(os.path.dirname(__file__), '*.yml')): +for f in glob(os.path.join(os.path.dirname(__file__), "*.yml")): MachineModel(path_to_yaml=f) - print('.', end='') + print(".", end="") sys.stdout.flush() # Iterating ISAs -for f in glob(os.path.join(os.path.dirname(__file__), 'isa/*.yml')): +for f in glob(os.path.join(os.path.dirname(__file__), "isa/*.yml")): MachineModel(path_to_yaml=f) - print('+', end='') + print("+", end="") sys.stdout.flush() -print() \ No newline at end of file +print() diff --git a/osaca/data/generate_mov_entries.py b/osaca/data/generate_mov_entries.py index 958f7c1..0c30a9d 100755 --- a/osaca/data/generate_mov_entries.py +++ b/osaca/data/generate_mov_entries.py @@ -14,8 +14,8 @@ class MOVEntryBuilder: @staticmethod def classify(operands_types): - load = 'mem' in operands_types[:-1] - store = 'mem' in operands_types[-1:] + load = "mem" in operands_types[:-1] + store = "mem" in operands_types[-1:] assert not (load and store), "Can not process a combined load-store instruction." return load, store @@ -26,28 +26,28 @@ class MOVEntryBuilder: comment = " # " + comment else: comment = "" - description = '- name: {}{}\n operands:\n'.format(instruction_name, comment) + description = "- name: {}{}\n operands:\n".format(instruction_name, comment) for ot in operand_types: - if ot == 'imd': - description += ' - class: immediate\n imd: int\n' - elif ot.startswith('mem'): - description += ' - class: memory\n' ' base: "*"\n' ' offset: "*"\n' - if ot == 'mem_simple': - description += ' index: ~\n' - elif ot == 'mem_complex': - description += ' index: gpr\n' + if ot == "imd": + description += " - class: immediate\n imd: int\n" + elif ot.startswith("mem"): + description += " - class: memory\n" ' base: "*"\n' ' offset: "*"\n' + if ot == "mem_simple": + description += " index: ~\n" + elif ot == "mem_complex": + description += " index: gpr\n" else: description += ' index: "*"\n' description += ' scale: "*"\n' else: - description += ' - class: register\n name: {}\n'.format(ot) + description += " - class: register\n name: {}\n".format(ot) description += ( - ' latency: {latency}\n' - ' port_pressure: {port_pressure!r}\n' - ' throughput: {throughput}\n' - ' uops: {uops}\n' + " latency: {latency}\n" + " port_pressure: {port_pressure!r}\n" + " throughput: {throughput}\n" + " uops: {uops}\n" ).format( latency=latency, port_pressure=port_pressure, @@ -63,8 +63,8 @@ class MOVEntryBuilder: """ port_pressure = [] if port_pressure_str: - for p in port_pressure_str.split('+'): - cycles, ports = p.split('*p') + for p in port_pressure_str.split("+"): + cycles, ports = p.split("*p") port_pressure.append([int(cycles), ports]) return port_pressure @@ -73,7 +73,7 @@ class MOVEntryBuilder: Example: ('mov xmm mem', ('1*p45+2*p0', 7) -> ('mov', ['xmm', 'mem'], [[1, '45'], [2, '0']], 7) """ - instr_elements = instruction_form.split(' ') + instr_elements = instruction_form.split(" ") latency = resources[1] port_pressure = self.parse_port_pressure(resources[0]) instruction_name = instr_elements[0] @@ -88,11 +88,11 @@ class MOVEntryBuilderIntelNoPort7AGU(MOVEntryBuilder): comment = None if load: - port_pressure += [[1, '23'], [1, ['2D', '3D']]] + port_pressure += [[1, "23"], [1, ["2D", "3D"]]] latency += 4 comment = "with load" if store: - port_pressure += [[1, '23'], [1, '4']] + port_pressure += [[1, "23"], [1, "4"]] latency += 0 comment = "with store" @@ -108,17 +108,17 @@ class MOVEntryBuilderIntelWithPort7AGU(MOVEntryBuilder): load, store = self.classify(operand_types) if load: - port_pressure += [[1, '23'], [1, ['2D', '3D']]] + port_pressure += [[1, "23"], [1, ["2D", "3D"]]] latency += 4 comment = "with load" return MOVEntryBuilder.build_description( self, instruction_name, operand_types, port_pressure, latency, comment ) if store: - port_pressure_simple = port_pressure + [[1, '237'], [1, '4']] - operands_simple = ['mem_simple' if o == 'mem' else o for o in operand_types] - port_pressure_complex = port_pressure + [[1, '23'], [1, '4']] - operands_complex = ['mem_complex' if o == 'mem' else o for o in operand_types] + port_pressure_simple = port_pressure + [[1, "237"], [1, "4"]] + operands_simple = ["mem_simple" if o == "mem" else o for o in operand_types] + port_pressure_complex = port_pressure + [[1, "23"], [1, "4"]] + operands_complex = ["mem_complex" if o == "mem" else o for o in operand_types] latency += 0 return ( MOVEntryBuilder.build_description( @@ -129,7 +129,7 @@ class MOVEntryBuilderIntelWithPort7AGU(MOVEntryBuilder): latency, "with store, simple AGU", ) - + '\n' + + "\n" + MOVEntryBuilder.build_description( self, instruction_name, @@ -152,319 +152,319 @@ p7 = MOVEntryBuilderIntelWithPort7AGU() # SNB snb_mov_instructions = [ # https://www.felixcloutier.com/x86/mov - ('mov gpr gpr', ('1*p015', 1)), - ('mov gpr mem', ('', 0)), - ('mov mem gpr', ('', 0)), - ('mov imd gpr', ('1*p015', 1)), - ('mov imd mem', ('', 0)), - ('movabs imd gpr', ('1*p015', 1)), # AT&T version + ("mov gpr gpr", ("1*p015", 1)), + ("mov gpr mem", ("", 0)), + ("mov mem gpr", ("", 0)), + ("mov imd gpr", ("1*p015", 1)), + ("mov imd mem", ("", 0)), + ("movabs imd gpr", ("1*p015", 1)), # AT&T version # https://www.felixcloutier.com/x86/movapd - ('movapd xmm xmm', ('1*p5', 1)), - ('movapd xmm mem', ('', 0)), - ('movapd mem xmm', ('', 0)), - ('vmovapd xmm xmm', ('1*p5', 1)), - ('vmovapd xmm mem', ('', 0)), - ('vmovapd mem xmm', ('', 0)), - ('vmovapd ymm ymm', ('1*p5', 1)), - ('vmovapd ymm mem', ('', 0)), - ('vmovapd mem ymm', ('', 0)), + ("movapd xmm xmm", ("1*p5", 1)), + ("movapd xmm mem", ("", 0)), + ("movapd mem xmm", ("", 0)), + ("vmovapd xmm xmm", ("1*p5", 1)), + ("vmovapd xmm mem", ("", 0)), + ("vmovapd mem xmm", ("", 0)), + ("vmovapd ymm ymm", ("1*p5", 1)), + ("vmovapd ymm mem", ("", 0)), + ("vmovapd mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movaps - ('movaps xmm xmm', ('1*p5', 1)), - ('movaps xmm mem', ('', 0)), - ('movaps mem xmm', ('', 0)), - ('vmovaps xmm xmm', ('1*p5', 1)), - ('movaps xmm mem', ('', 0)), - ('movaps mem xmm', ('', 0)), - ('vmovaps ymm ymm', ('1*p5', 1)), - ('movaps ymm mem', ('', 0)), - ('movaps mem ymm', ('', 0)), + ("movaps xmm xmm", ("1*p5", 1)), + ("movaps xmm mem", ("", 0)), + ("movaps mem xmm", ("", 0)), + ("vmovaps xmm xmm", ("1*p5", 1)), + ("movaps xmm mem", ("", 0)), + ("movaps mem xmm", ("", 0)), + ("vmovaps ymm ymm", ("1*p5", 1)), + ("movaps ymm mem", ("", 0)), + ("movaps mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movd:movq - ('movd gpr mm', ('1*p5', 1)), - ('movd mem mm', ('', 0)), - ('movq gpr mm', ('1*p5', 1)), - ('movq mem mm', ('', 0)), - ('movd mm gpr', ('1*p0', 1)), - ('movd mm mem', ('', 0)), - ('movq mm gpr', ('1*p0', 1)), - ('movq mm mem', ('', 0)), - ('movd gpr xmm', ('1*p5', 1)), - ('movd mem xmm', ('', 0)), - ('movq gpr xmm', ('1*p5', 1)), - ('movq mem xmm', ('', 0)), - ('movd xmm gpr', ('1*p0', 1)), - ('movd xmm mem', ('', 0)), - ('movq xmm gpr', ('1*p0', 1)), - ('movq xmm mem', ('', 0)), - ('vmovd gpr xmm', ('1*p5', 1)), - ('vmovd mem xmm', ('', 0)), - ('vmovq gpr xmm', ('1*p5', 1)), - ('vmovq mem xmm', ('', 0)), - ('vmovd xmm gpr', ('1*p0', 1)), - ('vmovd xmm mem', ('', 0)), - ('vmovq xmm gpr', ('1*p0', 1)), - ('vmovq xmm mem', ('', 0)), + ("movd gpr mm", ("1*p5", 1)), + ("movd mem mm", ("", 0)), + ("movq gpr mm", ("1*p5", 1)), + ("movq mem mm", ("", 0)), + ("movd mm gpr", ("1*p0", 1)), + ("movd mm mem", ("", 0)), + ("movq mm gpr", ("1*p0", 1)), + ("movq mm mem", ("", 0)), + ("movd gpr xmm", ("1*p5", 1)), + ("movd mem xmm", ("", 0)), + ("movq gpr xmm", ("1*p5", 1)), + ("movq mem xmm", ("", 0)), + ("movd xmm gpr", ("1*p0", 1)), + ("movd xmm mem", ("", 0)), + ("movq xmm gpr", ("1*p0", 1)), + ("movq xmm mem", ("", 0)), + ("vmovd gpr xmm", ("1*p5", 1)), + ("vmovd mem xmm", ("", 0)), + ("vmovq gpr xmm", ("1*p5", 1)), + ("vmovq mem xmm", ("", 0)), + ("vmovd xmm gpr", ("1*p0", 1)), + ("vmovd xmm mem", ("", 0)), + ("vmovq xmm gpr", ("1*p0", 1)), + ("vmovq xmm mem", ("", 0)), # https://www.felixcloutier.com/x86/movddup - ('movddup xmm xmm', ('1*p5', 1)), - ('movddup mem xmm', ('', 0)), - ('vmovddup xmm xmm', ('1*p5', 1)), - ('vmovddup mem xmm', ('', 0)), - ('vmovddup ymm ymm', ('1*p5', 1)), - ('vmovddup mem ymm', ('', 0)), + ("movddup xmm xmm", ("1*p5", 1)), + ("movddup mem xmm", ("", 0)), + ("vmovddup xmm xmm", ("1*p5", 1)), + ("vmovddup mem xmm", ("", 0)), + ("vmovddup ymm ymm", ("1*p5", 1)), + ("vmovddup mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movdq2q - ('movdq2q xmm mm', ('1*p015+1*p5', 1)), + ("movdq2q xmm mm", ("1*p015+1*p5", 1)), # https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64 - ('movdqa xmm xmm', ('1*p015', 1)), - ('movdqa mem xmm', ('', 0)), - ('movdqa xmm mem', ('', 0)), - ('vmovdqa xmm xmm', ('1*p015', 1)), - ('vmovdqa mem xmm', ('', 0)), - ('vmovdqa xmm mem', ('', 0)), - ('vmovdqa ymm ymm', ('1*p05', 1)), - ('vmovdqa mem ymm', ('', 0)), - ('vmovdqa ymm mem', ('', 0)), + ("movdqa xmm xmm", ("1*p015", 1)), + ("movdqa mem xmm", ("", 0)), + ("movdqa xmm mem", ("", 0)), + ("vmovdqa xmm xmm", ("1*p015", 1)), + ("vmovdqa mem xmm", ("", 0)), + ("vmovdqa xmm mem", ("", 0)), + ("vmovdqa ymm ymm", ("1*p05", 1)), + ("vmovdqa mem ymm", ("", 0)), + ("vmovdqa ymm mem", ("", 0)), # https://www.felixcloutier.com/x86/movdqu:vmovdqu8:vmovdqu16:vmovdqu32:vmovdqu64 - ('movdqu xmm xmm', ('1*p015', 1)), - ('movdqu mem xmm', ('', 0)), - ('movdqu xmm mem', ('', 0)), - ('vmovdqu xmm xmm', ('1*p015', 1)), - ('vmovdqu mem xmm', ('', 0)), - ('vmovdqu xmm mem', ('', 0)), - ('vmovdqu ymm ymm', ('1*p05', 1)), - ('vmovdqu mem ymm', ('', 0)), - ('vmovdqu ymm mem', ('', 0)), + ("movdqu xmm xmm", ("1*p015", 1)), + ("movdqu mem xmm", ("", 0)), + ("movdqu xmm mem", ("", 0)), + ("vmovdqu xmm xmm", ("1*p015", 1)), + ("vmovdqu mem xmm", ("", 0)), + ("vmovdqu xmm mem", ("", 0)), + ("vmovdqu ymm ymm", ("1*p05", 1)), + ("vmovdqu mem ymm", ("", 0)), + ("vmovdqu ymm mem", ("", 0)), # https://www.felixcloutier.com/x86/movhlps - ('movhlps xmm xmm', ('1*p5', 1)), - ('vmovhlps xmm xmm xmm', ('1*p5', 1)), + ("movhlps xmm xmm", ("1*p5", 1)), + ("vmovhlps xmm xmm xmm", ("1*p5", 1)), # https://www.felixcloutier.com/x86/movhpd - ('movhpd mem xmm', ('1*p5', 1)), - ('vmovhpd mem xmm xmm', ('1*p5', 1)), - ('movhpd xmm mem', ('', 0)), - ('vmovhpd mem xmm', ('', 0)), + ("movhpd mem xmm", ("1*p5", 1)), + ("vmovhpd mem xmm xmm", ("1*p5", 1)), + ("movhpd xmm mem", ("", 0)), + ("vmovhpd mem xmm", ("", 0)), # https://www.felixcloutier.com/x86/movhps - ('movhps mem xmm', ('1*p5', 1)), - ('vmovhps mem xmm xmm', ('1*p5', 1)), - ('movhps xmm mem', ('', 0)), - ('vmovhps mem xmm', ('', 0)), + ("movhps mem xmm", ("1*p5", 1)), + ("vmovhps mem xmm xmm", ("1*p5", 1)), + ("movhps xmm mem", ("", 0)), + ("vmovhps mem xmm", ("", 0)), # https://www.felixcloutier.com/x86/movlhps - ('movlhps xmm xmm', ('1*p5', 1)), - ('vmovlhps xmm xmm xmm', ('1*p5', 1)), + ("movlhps xmm xmm", ("1*p5", 1)), + ("vmovlhps xmm xmm xmm", ("1*p5", 1)), # https://www.felixcloutier.com/x86/movlpd - ('movlpd mem xmm', ('1*p5', 1)), - ('vmovlpd mem xmm xmm', ('1*p5', 1)), - ('movlpd xmm mem', ('', 0)), - ('vmovlpd mem xmm', ('1*p5', 1)), + ("movlpd mem xmm", ("1*p5", 1)), + ("vmovlpd mem xmm xmm", ("1*p5", 1)), + ("movlpd xmm mem", ("", 0)), + ("vmovlpd mem xmm", ("1*p5", 1)), # https://www.felixcloutier.com/x86/movlps - ('movlps mem xmm', ('1*p5', 1)), - ('vmovlps mem xmm xmm', ('1*p5', 1)), - ('movlps xmm mem', ('', 0)), - ('vmovlps mem xmm', ('1*p5', 1)), + ("movlps mem xmm", ("1*p5", 1)), + ("vmovlps mem xmm xmm", ("1*p5", 1)), + ("movlps xmm mem", ("", 0)), + ("vmovlps mem xmm", ("1*p5", 1)), # https://www.felixcloutier.com/x86/movmskpd - ('movmskpd xmm gpr', ('1*p0', 2)), - ('vmovmskpd xmm gpr', ('1*p0', 2)), - ('vmovmskpd ymm gpr', ('1*p0', 2)), + ("movmskpd xmm gpr", ("1*p0", 2)), + ("vmovmskpd xmm gpr", ("1*p0", 2)), + ("vmovmskpd ymm gpr", ("1*p0", 2)), # https://www.felixcloutier.com/x86/movmskps - ('movmskps xmm gpr', ('1*p0', 1)), - ('vmovmskps xmm gpr', ('1*p0', 1)), - ('vmovmskps ymm gpr', ('1*p0', 1)), + ("movmskps xmm gpr", ("1*p0", 1)), + ("vmovmskps xmm gpr", ("1*p0", 1)), + ("vmovmskps ymm gpr", ("1*p0", 1)), # https://www.felixcloutier.com/x86/movntdq - ('movntdq xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntdq xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntdq ymm mem', ('', 0)), # TODO NT-store: what latency to use? + ("movntdq xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntdq xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntdq ymm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntdqa - ('movntdqa mem xmm', ('', 0)), - ('vmovntdqa mem xmm', ('', 0)), - ('vmovntdqa mem ymm', ('', 0)), + ("movntdqa mem xmm", ("", 0)), + ("vmovntdqa mem xmm", ("", 0)), + ("vmovntdqa mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movnti - ('movnti gpr mem', ('', 0)), # TODO NT-store: what latency to use? + ("movnti gpr mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntpd - ('movntpd xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntpd xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntpd ymm mem', ('', 0)), # TODO NT-store: what latency to use? + ("movntpd xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntpd xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntpd ymm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntps - ('movntps xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntps xmm mem', ('', 0)), # TODO NT-store: what latency to use? - ('vmovntps ymm mem', ('', 0)), # TODO NT-store: what latency to use? + ("movntps xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntps xmm mem", ("", 0)), # TODO NT-store: what latency to use? + ("vmovntps ymm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntq - ('movntq mm mem', ('', 0)), # TODO NT-store: what latency to use? + ("movntq mm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movq - ('movq mm mm', ('', 0)), - ('movq mem mm', ('', 0)), - ('movq mm mem', ('', 0)), - ('movq xmm xmm', ('1*p015', 1)), - ('movq mem xmm', ('', 0)), - ('movq xmm mem', ('', 0)), - ('vmovq xmm xmm', ('1*p015', 1)), - ('vmovq mem xmm', ('', 0)), - ('vmovq xmm mem', ('', 0)), + ("movq mm mm", ("", 0)), + ("movq mem mm", ("", 0)), + ("movq mm mem", ("", 0)), + ("movq xmm xmm", ("1*p015", 1)), + ("movq mem xmm", ("", 0)), + ("movq xmm mem", ("", 0)), + ("vmovq xmm xmm", ("1*p015", 1)), + ("vmovq mem xmm", ("", 0)), + ("vmovq xmm mem", ("", 0)), # https://www.felixcloutier.com/x86/movq2dq - ('movq2dq mm xmm', ('1*p015', 1)), + ("movq2dq mm xmm", ("1*p015", 1)), # https://www.felixcloutier.com/x86/movs:movsb:movsw:movsd:movsq # TODO combined load-store is currently not supported # ('movs mem mem', ()), # https://www.felixcloutier.com/x86/movsd - ('movsd xmm xmm', ('1*p5', 1)), - ('movsd mem xmm', ('', 0)), - ('movsd xmm mem', ('', 0)), - ('vmovsd xmm xmm xmm', ('1*p5', 1)), - ('vmovsd mem xmm', ('', 0)), - ('vmovsd xmm mem', ('', 0)), + ("movsd xmm xmm", ("1*p5", 1)), + ("movsd mem xmm", ("", 0)), + ("movsd xmm mem", ("", 0)), + ("vmovsd xmm xmm xmm", ("1*p5", 1)), + ("vmovsd mem xmm", ("", 0)), + ("vmovsd xmm mem", ("", 0)), # https://www.felixcloutier.com/x86/movshdup - ('movshdup xmm xmm', ('1*p5', 1)), - ('movshdup mem xmm', ('', 0)), - ('vmovshdup xmm xmm', ('1*p5', 1)), - ('vmovshdup mem xmm', ('', 0)), - ('vmovshdup ymm ymm', ('1*p5', 1)), - ('vmovshdup mem ymm', ('', 0)), + ("movshdup xmm xmm", ("1*p5", 1)), + ("movshdup mem xmm", ("", 0)), + ("vmovshdup xmm xmm", ("1*p5", 1)), + ("vmovshdup mem xmm", ("", 0)), + ("vmovshdup ymm ymm", ("1*p5", 1)), + ("vmovshdup mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movsldup - ('movsldup xmm xmm', ('1*p5', 1)), - ('movsldup mem xmm', ('', 0)), - ('vmovsldup xmm xmm', ('1*p5', 1)), - ('vmovsldup mem xmm', ('', 0)), - ('vmovsldup ymm ymm', ('1*p5', 1)), - ('vmovsldup mem ymm', ('', 0)), + ("movsldup xmm xmm", ("1*p5", 1)), + ("movsldup mem xmm", ("", 0)), + ("vmovsldup xmm xmm", ("1*p5", 1)), + ("vmovsldup mem xmm", ("", 0)), + ("vmovsldup ymm ymm", ("1*p5", 1)), + ("vmovsldup mem ymm", ("", 0)), # https://www.felixcloutier.com/x86/movss - ('movss xmm xmm', ('1*p5', 1)), - ('movss mem xmm', ('', 0)), - ('vmovss xmm xmm xmm', ('1*p5', 1)), - ('vmovss mem xmm', ('', 0)), - ('vmovss xmm xmm', ('1*p5', 1)), - ('vmovss xmm mem', ('', 0)), - ('movss mem xmm', ('', 0)), + ("movss xmm xmm", ("1*p5", 1)), + ("movss mem xmm", ("", 0)), + ("vmovss xmm xmm xmm", ("1*p5", 1)), + ("vmovss mem xmm", ("", 0)), + ("vmovss xmm xmm", ("1*p5", 1)), + ("vmovss xmm mem", ("", 0)), + ("movss mem xmm", ("", 0)), # https://www.felixcloutier.com/x86/movsx:movsxd - ('movsx gpr gpr', ('1*p015', 1)), - ('movsx mem gpr', ('', 0)), - ('movsxd gpr gpr', ('', 0)), - ('movsxd mem gpr', ('', 0)), - ('movsb gpr gpr', ('1*p015', 1)), # AT&T version - ('movsb mem gpr', ('', 0)), # AT&T version - ('movsw gpr gpr', ('1*p015', 1)), # AT&T version - ('movsw mem gpr', ('', 0)), # AT&T version - ('movsl gpr gpr', ('1*p015', 1)), # AT&T version - ('movsl mem gpr', ('', 0)), # AT&T version - ('movsq gpr gpr', ('1*p015', 1)), # AT&T version - ('movsq mem gpr', ('', 0)), # AT&T version + ("movsx gpr gpr", ("1*p015", 1)), + ("movsx mem gpr", ("", 0)), + ("movsxd gpr gpr", ("", 0)), + ("movsxd mem gpr", ("", 0)), + ("movsb gpr gpr", ("1*p015", 1)), # AT&T version + ("movsb mem gpr", ("", 0)), # AT&T version + ("movsw gpr gpr", ("1*p015", 1)), # AT&T version + ("movsw mem gpr", ("", 0)), # AT&T version + ("movsl gpr gpr", ("1*p015", 1)), # AT&T version + ("movsl mem gpr", ("", 0)), # AT&T version + ("movsq gpr gpr", ("1*p015", 1)), # AT&T version + ("movsq mem gpr", ("", 0)), # AT&T version # https://www.felixcloutier.com/x86/movupd - ('movupd xmm xmm', ('1*p5', 1)), - ('movupd mem xmm', ('', 0)), - ('movupd xmm mem', ('', 0)), - ('vmovupd xmm xmm', ('1*p5', 1)), - ('vmovupd mem xmm', ('', 0)), - ('vmovupd xmm mem', ('', 0)), - ('vmovupd ymm ymm', ('1*p5', 1)), - ('vmovupd mem ymm', ('', 0)), - ('vmovupd ymm mem', ('', 0)), + ("movupd xmm xmm", ("1*p5", 1)), + ("movupd mem xmm", ("", 0)), + ("movupd xmm mem", ("", 0)), + ("vmovupd xmm xmm", ("1*p5", 1)), + ("vmovupd mem xmm", ("", 0)), + ("vmovupd xmm mem", ("", 0)), + ("vmovupd ymm ymm", ("1*p5", 1)), + ("vmovupd mem ymm", ("", 0)), + ("vmovupd ymm mem", ("", 0)), # https://www.felixcloutier.com/x86/movups - ('movups xmm xmm', ('1*p5', 1)), - ('movups mem xmm', ('', 0)), - ('movups xmm mem', ('', 0)), - ('vmovups xmm xmm', ('1*p5', 1)), - ('vmovups mem xmm', ('', 0)), - ('vmovups xmm mem', ('', 0)), - ('vmovups ymm ymm', ('1*p5', 1)), - ('vmovups mem ymm', ('', 0)), - ('vmovups ymm mem', ('', 0)), + ("movups xmm xmm", ("1*p5", 1)), + ("movups mem xmm", ("", 0)), + ("movups xmm mem", ("", 0)), + ("vmovups xmm xmm", ("1*p5", 1)), + ("vmovups mem xmm", ("", 0)), + ("vmovups xmm mem", ("", 0)), + ("vmovups ymm ymm", ("1*p5", 1)), + ("vmovups mem ymm", ("", 0)), + ("vmovups ymm mem", ("", 0)), # https://www.felixcloutier.com/x86/movzx - ('movzx gpr gpr', ('1*p015', 1)), - ('movzx mem gpr', ('', 0)), - ('movzb gpr gpr', ('1*p015', 1)), # AT&T version - ('movzb mem gpr', ('', 0)), # AT&T version - ('movzw gpr gpr', ('1*p015', 1)), # AT&T version - ('movzw mem gpr', ('', 0)), # AT&T version - ('movzl gpr gpr', ('1*p015', 1)), # AT&T version - ('movzl mem gpr', ('', 0)), # AT&T version - ('movzq gpr gpr', ('1*p015', 1)), # AT&T version - ('movzq mem gpr', ('', 0)), # AT&T version + ("movzx gpr gpr", ("1*p015", 1)), + ("movzx mem gpr", ("", 0)), + ("movzb gpr gpr", ("1*p015", 1)), # AT&T version + ("movzb mem gpr", ("", 0)), # AT&T version + ("movzw gpr gpr", ("1*p015", 1)), # AT&T version + ("movzw mem gpr", ("", 0)), # AT&T version + ("movzl gpr gpr", ("1*p015", 1)), # AT&T version + ("movzl mem gpr", ("", 0)), # AT&T version + ("movzq gpr gpr", ("1*p015", 1)), # AT&T version + ("movzq mem gpr", ("", 0)), # AT&T version # https://www.felixcloutier.com/x86/cmovcc - ('cmova gpr gpr', ('1*p015+2*p05', 2)), - ('cmova mem gpr', ('1*p015+2*p05', 2)), - ('cmovae gpr gpr', ('1*p015+1*p05', 2)), - ('cmovae mem gpr', ('1*p015+2*p05', 2)), - ('cmovb gpr gpr', ('1*p015+2*p05', 2)), - ('cmovb mem gpr', ('1*p015+1*p05', 2)), - ('cmovbe gpr gpr', ('1*p015+2*p05', 2)), - ('cmovbe mem gpr', ('1*p015+2*p05', 2)), - ('cmovc gpr gpr', ('1*p015+1*p05', 2)), - ('cmovc mem gpr', ('1*p015+1*p05', 2)), - ('cmove gpr gpr', ('1*p015+1*p05', 2)), - ('cmove mem gpr', ('1*p015+1*p05', 2)), - ('cmovg gpr gpr', ('1*p015+1*p05', 2)), - ('cmovg mem gpr', ('1*p015+1*p05', 2)), - ('cmovge gpr gpr', ('1*p015+1*p05', 2)), - ('cmovge mem gpr', ('1*p015+1*p05', 2)), - ('cmovl gpr gpr', ('1*p015+1*p05', 2)), - ('cmovl mem gpr', ('1*p015+1*p05', 2)), - ('cmovle gpr gpr', ('1*p015+1*p05', 2)), - ('cmovle mem gpr', ('1*p015+1*p05', 2)), - ('cmovna gpr gpr', ('1*p015+2*p05', 2)), - ('cmovna mem gpr', ('1*p015+2*p05', 2)), - ('cmovnae gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnae mem gpr', ('1*p015+1*p05', 2)), - ('cmovnb gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnb mem gpr', ('1*p015+1*p05', 2)), - ('cmovnbe gpr gpr', ('1*p015+2*p05', 2)), - ('cmovnbe mem gpr', ('1*p015+2*p05', 2)), - ('cmovnb gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnb mem gpr', ('1*p015+1*p05', 2)), - ('cmovnc gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnc mem gpr', ('1*p015+1*p05', 2)), - ('cmovne gpr gpr', ('1*p015+1*p05', 2)), - ('cmovne mem gpr', ('1*p015+1*p05', 2)), - ('cmovng gpr gpr', ('1*p015+1*p05', 2)), - ('cmovng mem gpr', ('1*p015+1*p05', 2)), - ('cmovnge gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnge mem gpr', ('1*p015+1*p05', 2)), - ('cmovnl gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnl mem gpr', ('1*p015+1*p05', 2)), - ('cmovno gpr gpr', ('1*p015+1*p05', 2)), - ('cmovno mem gpr', ('1*p015+1*p05', 2)), - ('cmovnp gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnp mem gpr', ('1*p015+1*p05', 2)), - ('cmovns gpr gpr', ('1*p015+1*p05', 2)), - ('cmovns mem gpr', ('1*p015+1*p05', 2)), - ('cmovnz gpr gpr', ('1*p015+1*p05', 2)), - ('cmovnz mem gpr', ('1*p015+1*p05', 2)), - ('cmovo gpr gpr', ('1*p015+1*p05', 2)), - ('cmovo mem gpr', ('1*p015+1*p05', 2)), - ('cmovp gpr gpr', ('1*p015+1*p05', 2)), - ('cmovp mem gpr', ('1*p015+1*p05', 2)), - ('cmovpe gpr gpr', ('1*p015+1*p05', 2)), - ('cmovpe mem gpr', ('1*p015+1*p05', 2)), - ('cmovpo gpr gpr', ('1*p015+1*p05', 2)), - ('cmovpo mem gpr', ('1*p015+1*p05', 2)), - ('cmovs gpr gpr', ('1*p015+1*p05', 2)), - ('cmovs mem gpr', ('1*p015+1*p05', 2)), - ('cmovz gpr gpr', ('1*p015+1*p05', 2)), - ('cmovz mem gpr', ('1*p015+1*p05', 2)), + ("cmova gpr gpr", ("1*p015+2*p05", 2)), + ("cmova mem gpr", ("1*p015+2*p05", 2)), + ("cmovae gpr gpr", ("1*p015+1*p05", 2)), + ("cmovae mem gpr", ("1*p015+2*p05", 2)), + ("cmovb gpr gpr", ("1*p015+2*p05", 2)), + ("cmovb mem gpr", ("1*p015+1*p05", 2)), + ("cmovbe gpr gpr", ("1*p015+2*p05", 2)), + ("cmovbe mem gpr", ("1*p015+2*p05", 2)), + ("cmovc gpr gpr", ("1*p015+1*p05", 2)), + ("cmovc mem gpr", ("1*p015+1*p05", 2)), + ("cmove gpr gpr", ("1*p015+1*p05", 2)), + ("cmove mem gpr", ("1*p015+1*p05", 2)), + ("cmovg gpr gpr", ("1*p015+1*p05", 2)), + ("cmovg mem gpr", ("1*p015+1*p05", 2)), + ("cmovge gpr gpr", ("1*p015+1*p05", 2)), + ("cmovge mem gpr", ("1*p015+1*p05", 2)), + ("cmovl gpr gpr", ("1*p015+1*p05", 2)), + ("cmovl mem gpr", ("1*p015+1*p05", 2)), + ("cmovle gpr gpr", ("1*p015+1*p05", 2)), + ("cmovle mem gpr", ("1*p015+1*p05", 2)), + ("cmovna gpr gpr", ("1*p015+2*p05", 2)), + ("cmovna mem gpr", ("1*p015+2*p05", 2)), + ("cmovnae gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnae mem gpr", ("1*p015+1*p05", 2)), + ("cmovnb gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnb mem gpr", ("1*p015+1*p05", 2)), + ("cmovnbe gpr gpr", ("1*p015+2*p05", 2)), + ("cmovnbe mem gpr", ("1*p015+2*p05", 2)), + ("cmovnb gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnb mem gpr", ("1*p015+1*p05", 2)), + ("cmovnc gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnc mem gpr", ("1*p015+1*p05", 2)), + ("cmovne gpr gpr", ("1*p015+1*p05", 2)), + ("cmovne mem gpr", ("1*p015+1*p05", 2)), + ("cmovng gpr gpr", ("1*p015+1*p05", 2)), + ("cmovng mem gpr", ("1*p015+1*p05", 2)), + ("cmovnge gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnge mem gpr", ("1*p015+1*p05", 2)), + ("cmovnl gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnl mem gpr", ("1*p015+1*p05", 2)), + ("cmovno gpr gpr", ("1*p015+1*p05", 2)), + ("cmovno mem gpr", ("1*p015+1*p05", 2)), + ("cmovnp gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnp mem gpr", ("1*p015+1*p05", 2)), + ("cmovns gpr gpr", ("1*p015+1*p05", 2)), + ("cmovns mem gpr", ("1*p015+1*p05", 2)), + ("cmovnz gpr gpr", ("1*p015+1*p05", 2)), + ("cmovnz mem gpr", ("1*p015+1*p05", 2)), + ("cmovo gpr gpr", ("1*p015+1*p05", 2)), + ("cmovo mem gpr", ("1*p015+1*p05", 2)), + ("cmovp gpr gpr", ("1*p015+1*p05", 2)), + ("cmovp mem gpr", ("1*p015+1*p05", 2)), + ("cmovpe gpr gpr", ("1*p015+1*p05", 2)), + ("cmovpe mem gpr", ("1*p015+1*p05", 2)), + ("cmovpo gpr gpr", ("1*p015+1*p05", 2)), + ("cmovpo mem gpr", ("1*p015+1*p05", 2)), + ("cmovs gpr gpr", ("1*p015+1*p05", 2)), + ("cmovs mem gpr", ("1*p015+1*p05", 2)), + ("cmovz gpr gpr", ("1*p015+1*p05", 2)), + ("cmovz mem gpr", ("1*p015+1*p05", 2)), # https://www.felixcloutier.com/x86/pmovmskb - ('pmovmskb mm gpr', ('1*p0', 2)), - ('pmovmskb xmm gpr', ('1*p0', 2)), - ('vpmovmskb xmm gpr', ('1*p0', 2)), + ("pmovmskb mm gpr", ("1*p0", 2)), + ("pmovmskb xmm gpr", ("1*p0", 2)), + ("vpmovmskb xmm gpr", ("1*p0", 2)), # https://www.felixcloutier.com/x86/pmovsx - ('pmovsxbw xmm xmm', ('1*p15', 1)), - ('pmovsxbw mem xmm', ('1*p15', 1)), - ('pmovsxbd xmm xmm', ('1*p15', 1)), - ('pmovsxbd mem xmm', ('1*p15', 1)), - ('pmovsxbq xmm xmm', ('1*p15', 1)), - ('pmovsxbq mem xmm', ('1*p15', 1)), - ('vpmovsxbw xmm xmm', ('1*p15', 1)), - ('vpmovsxbw mem xmm', ('1*p15', 1)), - ('vpmovsxbd xmm xmm', ('1*p15', 1)), - ('vpmovsxbd mem xmm', ('1*p15', 1)), - ('vpmovsxbq xmm xmm', ('1*p15', 1)), - ('vpmovsxbq mem xmm', ('1*p15', 1)), - ('vpmovsxbw ymm ymm', ('1*p15', 1)), - ('vpmovsxbw mem ymm', ('1*p15', 1)), - ('vpmovsxbd ymm ymm', ('1*p15', 1)), - ('vpmovsxbd mem ymm', ('1*p15', 1)), - ('vpmovsxbq ymm ymm', ('1*p15', 1)), - ('vpmovsxbq mem ymm', ('1*p15', 1)), + ("pmovsxbw xmm xmm", ("1*p15", 1)), + ("pmovsxbw mem xmm", ("1*p15", 1)), + ("pmovsxbd xmm xmm", ("1*p15", 1)), + ("pmovsxbd mem xmm", ("1*p15", 1)), + ("pmovsxbq xmm xmm", ("1*p15", 1)), + ("pmovsxbq mem xmm", ("1*p15", 1)), + ("vpmovsxbw xmm xmm", ("1*p15", 1)), + ("vpmovsxbw mem xmm", ("1*p15", 1)), + ("vpmovsxbd xmm xmm", ("1*p15", 1)), + ("vpmovsxbd mem xmm", ("1*p15", 1)), + ("vpmovsxbq xmm xmm", ("1*p15", 1)), + ("vpmovsxbq mem xmm", ("1*p15", 1)), + ("vpmovsxbw ymm ymm", ("1*p15", 1)), + ("vpmovsxbw mem ymm", ("1*p15", 1)), + ("vpmovsxbd ymm ymm", ("1*p15", 1)), + ("vpmovsxbd mem ymm", ("1*p15", 1)), + ("vpmovsxbq ymm ymm", ("1*p15", 1)), + ("vpmovsxbq mem ymm", ("1*p15", 1)), # https://www.felixcloutier.com/x86/pmovzx - ('pmovzxbw xmm xmm', ('1*p15', 1)), - ('pmovzxbw mem xmm', ('1*p15', 1)), - ('vpmovzxbw xmm xmm', ('1*p15', 1)), - ('vpmovzxbw mem xmm', ('1*p15', 1)), - ('vpmovzxbw ymm ymm', ('1*p15', 1)), - ('vpmovzxbw mem ymm', ('1*p15', 1)), + ("pmovzxbw xmm xmm", ("1*p15", 1)), + ("pmovzxbw mem xmm", ("1*p15", 1)), + ("vpmovzxbw xmm xmm", ("1*p15", 1)), + ("vpmovzxbw mem xmm", ("1*p15", 1)), + ("vpmovzxbw ymm ymm", ("1*p15", 1)), + ("vpmovzxbw mem ymm", ("1*p15", 1)), ] ivb_mov_instructions = list( @@ -472,32 +472,32 @@ ivb_mov_instructions = list( snb_mov_instructions + [ # https://www.felixcloutier.com/x86/mov - ('mov gpr gpr', ('', 0)), - ('mov imd gpr', ('', 0)), + ("mov gpr gpr", ("", 0)), + ("mov imd gpr", ("", 0)), # https://www.felixcloutier.com/x86/movapd - ('movapd xmm xmm', ('', 0)), - ('vmovapd xmm xmm', ('', 0)), - ('vmovapd ymm ymm', ('', 0)), + ("movapd xmm xmm", ("", 0)), + ("vmovapd xmm xmm", ("", 0)), + ("vmovapd ymm ymm", ("", 0)), # https://www.felixcloutier.com/x86/movaps - ('movaps xmm xmm', ('', 0)), - ('vmovaps xmm xmm', ('', 0)), - ('vmovaps ymm ymm', ('', 0)), + ("movaps xmm xmm", ("", 0)), + ("vmovaps xmm xmm", ("", 0)), + ("vmovaps ymm ymm", ("", 0)), # https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64 - ('movdqa xmm xmm', ('', 0)), - ('vmovdqa xmm xmm', ('', 0)), - ('vmovdqa ymm ymm', ('', 0)), + ("movdqa xmm xmm", ("", 0)), + ("vmovdqa xmm xmm", ("", 0)), + ("vmovdqa ymm ymm", ("", 0)), # https://www.felixcloutier.com/x86/movdqu:vmovdqu8:vmovdqu16:vmovdqu32:vmovdqu64 - ('movdqu xmm xmm', ('', 0)), - ('vmovdqu xmm xmm', ('', 0)), - ('vmovdqu ymm ymm', ('', 0)), + ("movdqu xmm xmm", ("", 0)), + ("vmovdqu xmm xmm", ("", 0)), + ("vmovdqu ymm ymm", ("", 0)), # https://www.felixcloutier.com/x86/movupd - ('movupd xmm xmm', ('', 0)), - ('vmovupd xmm xmm', ('', 0)), - ('vmovupd ymm ymm', ('', 0)), + ("movupd xmm xmm", ("", 0)), + ("vmovupd xmm xmm", ("", 0)), + ("vmovupd ymm ymm", ("", 0)), # https://www.felixcloutier.com/x86/movupd - ('movups xmm xmm', ('', 0)), - ('vmovups xmm xmm', ('', 0)), - ('vmovups ymm ymm', ('', 0)), + ("movups xmm xmm", ("", 0)), + ("vmovups xmm xmm", ("", 0)), + ("vmovups ymm ymm", ("", 0)), ] ).items() ) @@ -507,124 +507,124 @@ hsw_mov_instructions = list( ivb_mov_instructions + [ # https://www.felixcloutier.com/x86/mov - ('mov imd gpr', ('1*p0156', 1)), - ('mov gpr gpr', ('1*p0156', 1)), - ('movabs imd gpr', ('1*p0156', 1)), # AT&T version + ("mov imd gpr", ("1*p0156", 1)), + ("mov gpr gpr", ("1*p0156", 1)), + ("movabs imd gpr", ("1*p0156", 1)), # AT&T version # https://www.felixcloutier.com/x86/movbe - ('movbe gpr mem', ('1*p15', 6)), - ('movbe mem gpr', ('1*p15', 6)), + ("movbe gpr mem", ("1*p15", 6)), + ("movbe mem gpr", ("1*p15", 6)), # https://www.felixcloutier.com/x86/movmskpd - ('movmskpd xmm gpr', ('1*p0', 3)), - ('vmovmskpd xmm gpr', ('1*p0', 3)), - ('vmovmskpd ymm gpr', ('1*p0', 3)), + ("movmskpd xmm gpr", ("1*p0", 3)), + ("vmovmskpd xmm gpr", ("1*p0", 3)), + ("vmovmskpd ymm gpr", ("1*p0", 3)), # https://www.felixcloutier.com/x86/movmskps - ('movmskps xmm gpr', ('1*p0', 3)), - ('vmovmskps xmm gpr', ('1*p0', 3)), - ('vmovmskps ymm gpr', ('1*p0', 3)), + ("movmskps xmm gpr", ("1*p0", 3)), + ("vmovmskps xmm gpr", ("1*p0", 3)), + ("vmovmskps ymm gpr", ("1*p0", 3)), # https://www.felixcloutier.com/x86/movsx:movsxd - ('movsx gpr gpr', ('1*p0156', 1)), - ('movsb gpr gpr', ('1*p0156', 1)), # AT&T version - ('movsw gpr gpr', ('1*p0156', 1)), # AT&T version - ('movsl gpr gpr', ('1*p0156', 1)), # AT&T version - ('movsq gpr gpr', ('1*p0156', 1)), # AT&T version + ("movsx gpr gpr", ("1*p0156", 1)), + ("movsb gpr gpr", ("1*p0156", 1)), # AT&T version + ("movsw gpr gpr", ("1*p0156", 1)), # AT&T version + ("movsl gpr gpr", ("1*p0156", 1)), # AT&T version + ("movsq gpr gpr", ("1*p0156", 1)), # AT&T version # https://www.felixcloutier.com/x86/movzx - ('movzx gpr gpr', ('1*p0156', 1)), - ('movzb gpr gpr', ('1*p0156', 1)), # AT&T version - ('movzw gpr gpr', ('1*p0156', 1)), # AT&T version - ('movzl gpr gpr', ('1*p0156', 1)), # AT&T version - ('movzq gpr gpr', ('1*p0156', 1)), # AT&T version + ("movzx gpr gpr", ("1*p0156", 1)), + ("movzb gpr gpr", ("1*p0156", 1)), # AT&T version + ("movzw gpr gpr", ("1*p0156", 1)), # AT&T version + ("movzl gpr gpr", ("1*p0156", 1)), # AT&T version + ("movzq gpr gpr", ("1*p0156", 1)), # AT&T version # https://www.felixcloutier.com/x86/cmovcc - ('cmova gpr gpr', ('1*p0156+2*p06', 2)), - ('cmova mem gpr', ('1*p0156+2*p06', 2)), - ('cmovae gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovae mem gpr', ('1*p0156+2*p06', 2)), - ('cmovb gpr gpr', ('1*p0156+2*p06', 2)), - ('cmovb mem gpr', ('1*p0156+1*p06', 2)), - ('cmovbe gpr gpr', ('1*p0156+2*p06', 2)), - ('cmovbe mem gpr', ('1*p0156+2*p06', 2)), - ('cmovc gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovc mem gpr', ('1*p0156+1*p06', 2)), - ('cmove gpr gpr', ('1*p0156+1*p06', 2)), - ('cmove mem gpr', ('1*p0156+1*p06', 2)), - ('cmovg gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovg mem gpr', ('1*p0156+1*p06', 2)), - ('cmovge gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovge mem gpr', ('1*p0156+1*p06', 2)), - ('cmovl gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovl mem gpr', ('1*p0156+1*p06', 2)), - ('cmovle gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovle mem gpr', ('1*p0156+1*p06', 2)), - ('cmovna gpr gpr', ('1*p0156+2*p06', 2)), - ('cmovna mem gpr', ('1*p0156+2*p06', 2)), - ('cmovnae gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnae mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnb gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnb mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnbe gpr gpr', ('1*p0156+2*p06', 2)), - ('cmovnbe mem gpr', ('1*p0156+2*p06', 2)), - ('cmovnb gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnb mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnc gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnc mem gpr', ('1*p0156+1*p06', 2)), - ('cmovne gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovne mem gpr', ('1*p0156+1*p06', 2)), - ('cmovng gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovng mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnge gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnge mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnl gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnl mem gpr', ('1*p0156+1*p06', 2)), - ('cmovno gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovno mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnp gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnp mem gpr', ('1*p0156+1*p06', 2)), - ('cmovns gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovns mem gpr', ('1*p0156+1*p06', 2)), - ('cmovnz gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovnz mem gpr', ('1*p0156+1*p06', 2)), - ('cmovo gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovo mem gpr', ('1*p0156+1*p06', 2)), - ('cmovp gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovp mem gpr', ('1*p0156+1*p06', 2)), - ('cmovpe gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovpe mem gpr', ('1*p0156+1*p06', 2)), - ('cmovpo gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovpo mem gpr', ('1*p0156+1*p06', 2)), - ('cmovs gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovs mem gpr', ('1*p0156+1*p06', 2)), - ('cmovz gpr gpr', ('1*p0156+1*p06', 2)), - ('cmovz mem gpr', ('1*p0156+1*p06', 2)), + ("cmova gpr gpr", ("1*p0156+2*p06", 2)), + ("cmova mem gpr", ("1*p0156+2*p06", 2)), + ("cmovae gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovae mem gpr", ("1*p0156+2*p06", 2)), + ("cmovb gpr gpr", ("1*p0156+2*p06", 2)), + ("cmovb mem gpr", ("1*p0156+1*p06", 2)), + ("cmovbe gpr gpr", ("1*p0156+2*p06", 2)), + ("cmovbe mem gpr", ("1*p0156+2*p06", 2)), + ("cmovc gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovc mem gpr", ("1*p0156+1*p06", 2)), + ("cmove gpr gpr", ("1*p0156+1*p06", 2)), + ("cmove mem gpr", ("1*p0156+1*p06", 2)), + ("cmovg gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovg mem gpr", ("1*p0156+1*p06", 2)), + ("cmovge gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovge mem gpr", ("1*p0156+1*p06", 2)), + ("cmovl gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovl mem gpr", ("1*p0156+1*p06", 2)), + ("cmovle gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovle mem gpr", ("1*p0156+1*p06", 2)), + ("cmovna gpr gpr", ("1*p0156+2*p06", 2)), + ("cmovna mem gpr", ("1*p0156+2*p06", 2)), + ("cmovnae gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnae mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnb gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnb mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnbe gpr gpr", ("1*p0156+2*p06", 2)), + ("cmovnbe mem gpr", ("1*p0156+2*p06", 2)), + ("cmovnb gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnb mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnc gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnc mem gpr", ("1*p0156+1*p06", 2)), + ("cmovne gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovne mem gpr", ("1*p0156+1*p06", 2)), + ("cmovng gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovng mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnge gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnge mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnl gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnl mem gpr", ("1*p0156+1*p06", 2)), + ("cmovno gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovno mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnp gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnp mem gpr", ("1*p0156+1*p06", 2)), + ("cmovns gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovns mem gpr", ("1*p0156+1*p06", 2)), + ("cmovnz gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovnz mem gpr", ("1*p0156+1*p06", 2)), + ("cmovo gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovo mem gpr", ("1*p0156+1*p06", 2)), + ("cmovp gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovp mem gpr", ("1*p0156+1*p06", 2)), + ("cmovpe gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovpe mem gpr", ("1*p0156+1*p06", 2)), + ("cmovpo gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovpo mem gpr", ("1*p0156+1*p06", 2)), + ("cmovs gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovs mem gpr", ("1*p0156+1*p06", 2)), + ("cmovz gpr gpr", ("1*p0156+1*p06", 2)), + ("cmovz mem gpr", ("1*p0156+1*p06", 2)), # https://www.felixcloutier.com/x86/pmovmskb - ('pmovmskb mm gpr', ('1*p0', 3)), - ('pmovmskb xmm gpr', ('1*p0', 3)), - ('vpmovmskb xmm gpr', ('1*p0', 3)), - ('vpmovmskb ymm gpr', ('1*p0', 3)), + ("pmovmskb mm gpr", ("1*p0", 3)), + ("pmovmskb xmm gpr", ("1*p0", 3)), + ("vpmovmskb xmm gpr", ("1*p0", 3)), + ("vpmovmskb ymm gpr", ("1*p0", 3)), # https://www.felixcloutier.com/x86/pmovsx - ('pmovsxbw xmm xmm', ('1*p5', 1)), - ('pmovsxbw mem xmm', ('1*p5', 1)), - ('pmovsxbd xmm xmm', ('1*p5', 1)), - ('pmovsxbd mem xmm', ('1*p5', 1)), - ('pmovsxbq xmm xmm', ('1*p5', 1)), - ('pmovsxbq mem xmm', ('1*p5', 1)), - ('vpmovsxbw xmm xmm', ('1*p5', 1)), - ('vpmovsxbw mem xmm', ('1*p5', 1)), - ('vpmovsxbd xmm xmm', ('1*p5', 1)), - ('vpmovsxbd mem xmm', ('1*p5', 1)), - ('vpmovsxbq xmm xmm', ('1*p5', 1)), - ('vpmovsxbq mem xmm', ('1*p5', 1)), - ('vpmovsxbw ymm ymm', ('1*p5', 1)), - ('vpmovsxbw mem ymm', ('1*p5', 1)), - ('vpmovsxbd ymm ymm', ('1*p5', 1)), - ('vpmovsxbd mem ymm', ('1*p5', 1)), - ('vpmovsxbq ymm ymm', ('1*p5', 1)), - ('vpmovsxbq mem ymm', ('1*p5', 1)), + ("pmovsxbw xmm xmm", ("1*p5", 1)), + ("pmovsxbw mem xmm", ("1*p5", 1)), + ("pmovsxbd xmm xmm", ("1*p5", 1)), + ("pmovsxbd mem xmm", ("1*p5", 1)), + ("pmovsxbq xmm xmm", ("1*p5", 1)), + ("pmovsxbq mem xmm", ("1*p5", 1)), + ("vpmovsxbw xmm xmm", ("1*p5", 1)), + ("vpmovsxbw mem xmm", ("1*p5", 1)), + ("vpmovsxbd xmm xmm", ("1*p5", 1)), + ("vpmovsxbd mem xmm", ("1*p5", 1)), + ("vpmovsxbq xmm xmm", ("1*p5", 1)), + ("vpmovsxbq mem xmm", ("1*p5", 1)), + ("vpmovsxbw ymm ymm", ("1*p5", 1)), + ("vpmovsxbw mem ymm", ("1*p5", 1)), + ("vpmovsxbd ymm ymm", ("1*p5", 1)), + ("vpmovsxbd mem ymm", ("1*p5", 1)), + ("vpmovsxbq ymm ymm", ("1*p5", 1)), + ("vpmovsxbq mem ymm", ("1*p5", 1)), # https://www.felixcloutier.com/x86/pmovzx - ('pmovzxbw xmm xmm', ('1*p5', 1)), - ('pmovzxbw mem xmm', ('1*p5', 1)), - ('vpmovzxbw xmm xmm', ('1*p5', 1)), - ('vpmovzxbw mem xmm', ('1*p5', 1)), - ('vpmovzxbw ymm ymm', ('1*p5', 1)), - ('vpmovzxbw mem ymm', ('1*p5', 1)), + ("pmovzxbw xmm xmm", ("1*p5", 1)), + ("pmovzxbw mem xmm", ("1*p5", 1)), + ("vpmovzxbw xmm xmm", ("1*p5", 1)), + ("vpmovzxbw mem xmm", ("1*p5", 1)), + ("vpmovzxbw ymm ymm", ("1*p5", 1)), + ("vpmovzxbw mem ymm", ("1*p5", 1)), ] ).items() ) @@ -634,66 +634,66 @@ bdw_mov_instructions = list( hsw_mov_instructions + [ # https://www.felixcloutier.com/x86/cmovcc - ('cmova gpr gpr', ('2*p06', 1)), - ('cmova mem gpr', ('2*p06', 1)), - ('cmovae gpr gpr', ('1*p06', 1)), - ('cmovae mem gpr', ('2*p06', 1)), - ('cmovb gpr gpr', ('2*p06', 1)), - ('cmovb mem gpr', ('1*p06', 1)), - ('cmovbe gpr gpr', ('2*p06', 1)), - ('cmovbe mem gpr', ('2*p06', 1)), - ('cmovc gpr gpr', ('1*p06', 1)), - ('cmovc mem gpr', ('1*p06', 1)), - ('cmove gpr gpr', ('1*p06', 1)), - ('cmove mem gpr', ('1*p06', 1)), - ('cmovg gpr gpr', ('1*p06', 1)), - ('cmovg mem gpr', ('1*p06', 1)), - ('cmovge gpr gpr', ('1*p06', 1)), - ('cmovge mem gpr', ('1*p06', 1)), - ('cmovl gpr gpr', ('1*p06', 1)), - ('cmovl mem gpr', ('1*p06', 1)), - ('cmovle gpr gpr', ('1*p06', 1)), - ('cmovle mem gpr', ('1*p06', 1)), - ('cmovna gpr gpr', ('2*p06', 1)), - ('cmovna mem gpr', ('2*p06', 1)), - ('cmovnae gpr gpr', ('1*p06', 1)), - ('cmovnae mem gpr', ('1*p06', 1)), - ('cmovnb gpr gpr', ('1*p06', 1)), - ('cmovnb mem gpr', ('1*p06', 1)), - ('cmovnbe gpr gpr', ('2*p06', 1)), - ('cmovnbe mem gpr', ('2*p06', 1)), - ('cmovnb gpr gpr', ('1*p06', 1)), - ('cmovnb mem gpr', ('1*p06', 1)), - ('cmovnc gpr gpr', ('1*p06', 1)), - ('cmovnc mem gpr', ('1*p06', 1)), - ('cmovne gpr gpr', ('1*p06', 1)), - ('cmovne mem gpr', ('1*p06', 1)), - ('cmovng gpr gpr', ('1*p06', 1)), - ('cmovng mem gpr', ('1*p06', 1)), - ('cmovnge gpr gpr', ('1*p06', 1)), - ('cmovnge mem gpr', ('1*p06', 1)), - ('cmovnl gpr gpr', ('1*p06', 1)), - ('cmovnl mem gpr', ('1*p06', 1)), - ('cmovno gpr gpr', ('1*p06', 1)), - ('cmovno mem gpr', ('1*p06', 1)), - ('cmovnp gpr gpr', ('1*p06', 1)), - ('cmovnp mem gpr', ('1*p06', 1)), - ('cmovns gpr gpr', ('1*p06', 1)), - ('cmovns mem gpr', ('1*p06', 1)), - ('cmovnz gpr gpr', ('1*p06', 1)), - ('cmovnz mem gpr', ('1*p06', 1)), - ('cmovo gpr gpr', ('1*p06', 1)), - ('cmovo mem gpr', ('1*p06', 1)), - ('cmovp gpr gpr', ('1*p06', 1)), - ('cmovp mem gpr', ('1*p06', 1)), - ('cmovpe gpr gpr', ('1*p06', 1)), - ('cmovpe mem gpr', ('1*p06', 1)), - ('cmovpo gpr gpr', ('1*p06', 1)), - ('cmovpo mem gpr', ('1*p06', 1)), - ('cmovs gpr gpr', ('1*p06', 1)), - ('cmovs mem gpr', ('1*p06', 1)), - ('cmovz gpr gpr', ('1*p06', 1)), - ('cmovz mem gpr', ('1*p06', 1)), + ("cmova gpr gpr", ("2*p06", 1)), + ("cmova mem gpr", ("2*p06", 1)), + ("cmovae gpr gpr", ("1*p06", 1)), + ("cmovae mem gpr", ("2*p06", 1)), + ("cmovb gpr gpr", ("2*p06", 1)), + ("cmovb mem gpr", ("1*p06", 1)), + ("cmovbe gpr gpr", ("2*p06", 1)), + ("cmovbe mem gpr", ("2*p06", 1)), + ("cmovc gpr gpr", ("1*p06", 1)), + ("cmovc mem gpr", ("1*p06", 1)), + ("cmove gpr gpr", ("1*p06", 1)), + ("cmove mem gpr", ("1*p06", 1)), + ("cmovg gpr gpr", ("1*p06", 1)), + ("cmovg mem gpr", ("1*p06", 1)), + ("cmovge gpr gpr", ("1*p06", 1)), + ("cmovge mem gpr", ("1*p06", 1)), + ("cmovl gpr gpr", ("1*p06", 1)), + ("cmovl mem gpr", ("1*p06", 1)), + ("cmovle gpr gpr", ("1*p06", 1)), + ("cmovle mem gpr", ("1*p06", 1)), + ("cmovna gpr gpr", ("2*p06", 1)), + ("cmovna mem gpr", ("2*p06", 1)), + ("cmovnae gpr gpr", ("1*p06", 1)), + ("cmovnae mem gpr", ("1*p06", 1)), + ("cmovnb gpr gpr", ("1*p06", 1)), + ("cmovnb mem gpr", ("1*p06", 1)), + ("cmovnbe gpr gpr", ("2*p06", 1)), + ("cmovnbe mem gpr", ("2*p06", 1)), + ("cmovnb gpr gpr", ("1*p06", 1)), + ("cmovnb mem gpr", ("1*p06", 1)), + ("cmovnc gpr gpr", ("1*p06", 1)), + ("cmovnc mem gpr", ("1*p06", 1)), + ("cmovne gpr gpr", ("1*p06", 1)), + ("cmovne mem gpr", ("1*p06", 1)), + ("cmovng gpr gpr", ("1*p06", 1)), + ("cmovng mem gpr", ("1*p06", 1)), + ("cmovnge gpr gpr", ("1*p06", 1)), + ("cmovnge mem gpr", ("1*p06", 1)), + ("cmovnl gpr gpr", ("1*p06", 1)), + ("cmovnl mem gpr", ("1*p06", 1)), + ("cmovno gpr gpr", ("1*p06", 1)), + ("cmovno mem gpr", ("1*p06", 1)), + ("cmovnp gpr gpr", ("1*p06", 1)), + ("cmovnp mem gpr", ("1*p06", 1)), + ("cmovns gpr gpr", ("1*p06", 1)), + ("cmovns mem gpr", ("1*p06", 1)), + ("cmovnz gpr gpr", ("1*p06", 1)), + ("cmovnz mem gpr", ("1*p06", 1)), + ("cmovo gpr gpr", ("1*p06", 1)), + ("cmovo mem gpr", ("1*p06", 1)), + ("cmovp gpr gpr", ("1*p06", 1)), + ("cmovp mem gpr", ("1*p06", 1)), + ("cmovpe gpr gpr", ("1*p06", 1)), + ("cmovpe mem gpr", ("1*p06", 1)), + ("cmovpo gpr gpr", ("1*p06", 1)), + ("cmovpo mem gpr", ("1*p06", 1)), + ("cmovs gpr gpr", ("1*p06", 1)), + ("cmovs mem gpr", ("1*p06", 1)), + ("cmovz gpr gpr", ("1*p06", 1)), + ("cmovz mem gpr", ("1*p06", 1)), ] ).items() ) @@ -715,8 +715,8 @@ skx_mov_instructions = list( # ('vmovaps xmm xmm', ('1*p5', 1)), # ('vmovaps ymm ymm', ('1*p5', 1)), # https://www.felixcloutier.com/x86/movbe - ('movbe gpr mem', ('1*p15', 4)), - ('movbe mem gpr', ('1*p15', 4)), + ("movbe gpr mem", ("1*p15", 4)), + ("movbe mem gpr", ("1*p15", 4)), # https://www.felixcloutier.com/x86/movddup # TODO with masking! # https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64 @@ -724,15 +724,15 @@ skx_mov_instructions = list( # https://www.felixcloutier.com/x86/movdqu:vmovdqu8:vmovdqu16:vmovdqu32:vmovdqu64 # TODO with masking! # https://www.felixcloutier.com/x86/movntdq - ('vmovntdq zmm mem', ('', 0)), # TODO NT-store: what latency to use? + ("vmovntdq zmm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntdqa - ('vmovntdqa mem zmm', ('', 0)), + ("vmovntdqa mem zmm", ("", 0)), # https://www.felixcloutier.com/x86/movntpd - ('vmovntpd zmm mem', ('', 0)), # TODO NT-store: what latency to use? + ("vmovntpd zmm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movntps - ('vmovntps zmm mem', ('', 0)), # TODO NT-store: what latency to use? + ("vmovntps zmm mem", ("", 0)), # TODO NT-store: what latency to use? # https://www.felixcloutier.com/x86/movq2dq - ('movq2dq mm xmm', ('1*p0+1*p015', 1)), + ("movq2dq mm xmm", ("1*p0+1*p015", 1)), # https://www.felixcloutier.com/x86/movsd # TODO with masking! # https://www.felixcloutier.com/x86/movshdup @@ -747,8 +747,8 @@ skx_mov_instructions = list( # TODO with masking! # https://www.felixcloutier.com/x86/pmovsx # TODO with masking! - ('vpmovsxbw ymm zmm', ('1*p5', 3)), - ('vpmovsxbw mem zmm', ('1*p5', 1)), + ("vpmovsxbw ymm zmm", ("1*p5", 3)), + ("vpmovsxbw mem zmm", ("1*p5", 1)), ] ).items() ) @@ -758,28 +758,28 @@ csx_mov_instructions = OrderedDict(skx_mov_instructions + []).items() def get_description(arch, rhs_comment=None): descriptions = { - 'snb': '\n'.join([np7.process_item(*item) for item in snb_mov_instructions]), - 'ivb': '\n'.join([np7.process_item(*item) for item in ivb_mov_instructions]), - 'hsw': '\n'.join([p7.process_item(*item) for item in hsw_mov_instructions]), - 'bdw': '\n'.join([p7.process_item(*item) for item in bdw_mov_instructions]), - 'skx': '\n'.join([p7.process_item(*item) for item in skx_mov_instructions]), - 'csx': '\n'.join([p7.process_item(*item) for item in csx_mov_instructions]), + "snb": "\n".join([np7.process_item(*item) for item in snb_mov_instructions]), + "ivb": "\n".join([np7.process_item(*item) for item in ivb_mov_instructions]), + "hsw": "\n".join([p7.process_item(*item) for item in hsw_mov_instructions]), + "bdw": "\n".join([p7.process_item(*item) for item in bdw_mov_instructions]), + "skx": "\n".join([p7.process_item(*item) for item in skx_mov_instructions]), + "csx": "\n".join([p7.process_item(*item) for item in csx_mov_instructions]), } description = descriptions[arch] if rhs_comment is not None: - max_length = max([len(l) for l in descriptions[arch].split('\n')]) + max_length = max([len(l) for l in descriptions[arch].split("\n")]) commented_description = "" - for l in descriptions[arch].split('\n'): + for l in descriptions[arch].split("\n"): commented_description += ("{:<" + str(max_length) + "} # {}\n").format(l, rhs_comment) description = commented_description return description -if __name__ == '__main__': +if __name__ == "__main__": import sys if len(sys.argv) != 2: @@ -787,7 +787,7 @@ if __name__ == '__main__': sys.exit(0) try: - print(get_description(sys.argv[1], rhs_comment=' '.join(sys.argv))) + print(get_description(sys.argv[1], rhs_comment=" ".join(sys.argv))) except KeyError: print("Unknown architecture.") sys.exit(1) diff --git a/osaca/data/model_importer.py b/osaca/data/model_importer.py index 1a8068e..b45a01f 100755 --- a/osaca/data/model_importer.py +++ b/osaca/data/model_importer.py @@ -9,37 +9,37 @@ from osaca.parser import get_parser from osaca.semantics import MachineModel intel_archs = [ - 'CON', - 'WOL', - 'NHM', - 'WSM', - 'SNB', - 'IVB', - 'HSW', - 'BDW', - 'SKL', - 'SKX', - 'KBL', - 'CFL', - 'CNL', - 'ICL', + "CON", + "WOL", + "NHM", + "WSM", + "SNB", + "IVB", + "HSW", + "BDW", + "SKL", + "SKX", + "KBL", + "CFL", + "CNL", + "ICL", ] -amd_archs = ['ZEN1', 'ZEN+', 'ZEN2'] +amd_archs = ["ZEN1", "ZEN+", "ZEN2"] def port_pressure_from_tag_attributes(attrib): # '1*p015+1*p1+1*p23+1*p4+3*p5' -> # [[1, '015'], [1, '1'], [1, '23'], [1, '4'], [3, '5']] port_occupation = [] - for p in attrib['ports'].split('+'): - cycles, ports = p.split('*') - ports = ports.lstrip('p') - ports = ports.lstrip('FP') + for p in attrib["ports"].split("+"): + cycles, ports = p.split("*") + ports = ports.lstrip("p") + ports = ports.lstrip("FP") port_occupation.append([int(cycles), ports]) # Also consider div on DIV pipeline - if 'div_cycles' in attrib: - port_occupation.append([int(attrib['div_cycles']), ['DIV']]) + if "div_cycles" in attrib: + port_occupation.append([int(attrib["div_cycles"]), ["DIV"]]) return port_occupation @@ -47,57 +47,57 @@ def port_pressure_from_tag_attributes(attrib): def extract_paramters(instruction_tag, parser, isa): # Extract parameter components parameters = [] # used to store string representations - parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib['idx'])) + parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib["idx"])) for parameter_tag in parameter_tags: parameter = {} # Ignore parameters with suppressed=1 - if int(parameter_tag.attrib.get('suppressed', '0')): + if int(parameter_tag.attrib.get("suppressed", "0")): continue - p_type = parameter_tag.attrib['type'] - if p_type == 'imm': - parameter['class'] = 'immediate' - parameter['imd'] = 'int' + p_type = parameter_tag.attrib["type"] + if p_type == "imm": + parameter["class"] = "immediate" + parameter["imd"] = "int" parameters.append(parameter) - elif p_type == 'mem': - parameter['class'] = 'memory' - parameter['base'] = "*" - parameter['offset'] = "*" - parameter['index'] = "*" - parameter['scale'] = "*" + elif p_type == "mem": + parameter["class"] = "memory" + parameter["base"] = "*" + parameter["offset"] = "*" + parameter["index"] = "*" + parameter["scale"] = "*" parameters.append(parameter) - elif p_type == 'reg': - parameter['class'] = 'register' - possible_regs = [parser.parse_register('%' + r) for r in parameter_tag.text.split(',')] + elif p_type == "reg": + parameter["class"] = "register" + possible_regs = [parser.parse_register("%" + r) for r in parameter_tag.text.split(",")] if possible_regs[0] is None: raise ValueError( - 'Unknown register type for {} with {}.'.format( + "Unknown register type for {} with {}.".format( parameter_tag.attrib, parameter_tag.text ) ) - if isa == 'x86': - if parser.is_vector_register(possible_regs[0]['register']): - possible_regs[0]['register']['name'] = possible_regs[0]['register'][ - 'name' + if isa == "x86": + if parser.is_vector_register(possible_regs[0]["register"]): + possible_regs[0]["register"]["name"] = possible_regs[0]["register"][ + "name" ].lower()[:3] - if 'mask' in possible_regs[0]['register']: - possible_regs[0]['register']['mask'] = True + if "mask" in possible_regs[0]["register"]: + possible_regs[0]["register"]["mask"] = True else: - possible_regs[0]['register']['name'] = 'gpr' - elif isa == 'aarch64': - del possible_regs['register']['name'] - for key in possible_regs[0]['register']: - parameter[key] = possible_regs[0]['register'][key] + possible_regs[0]["register"]["name"] = "gpr" + elif isa == "aarch64": + del possible_regs["register"]["name"] + for key in possible_regs[0]["register"]: + parameter[key] = possible_regs[0]["register"][key] parameters.append(parameter) - elif p_type == 'relbr': - parameter['class'] = 'identifier' + elif p_type == "relbr": + parameter["class"] = "identifier" parameters.append(parameter) - elif p_type == 'agen': - parameter['class'] = 'memory' - parameter['base'] = "*" - parameter['offset'] = "*" - parameter['index'] = "*" - parameter['scale'] = "*" + elif p_type == "agen": + parameter["class"] = "memory" + parameter["base"] = "*" + parameter["offset"] = "*" + parameter["index"] = "*" + parameter["scale"] = "*" parameters.append(parameter) else: raise ValueError("Unknown paramter type {}".format(parameter_tag.attrib)) @@ -113,19 +113,19 @@ def extract_model(tree, arch, skip_mem=True): mm = MachineModel(isa=isa) parser = get_parser(isa) - for instruction_tag in tree.findall('.//instruction'): + for instruction_tag in tree.findall(".//instruction"): ignore = False - mnemonic = instruction_tag.attrib['asm'] - iform = instruction_tag.attrib['iform'] + mnemonic = instruction_tag.attrib["asm"] + iform = instruction_tag.attrib["iform"] # skip any mnemonic which contain spaces (e.g., "REX CRC32") - if ' ' in mnemonic: + if " " in mnemonic: continue # Extract parameter components try: parameters = extract_paramters(instruction_tag, parser, isa) - if isa == 'x86': + if isa == "x86": parameters.reverse() except ValueError as e: print(e, file=sys.stderr) @@ -136,13 +136,11 @@ def extract_model(tree, arch, skip_mem=True): if arch_tag is None: continue # skip any instructions without port utilization - if not any(['ports' in x.attrib for x in arch_tag.findall('measurement')]): + if not any(["ports" in x.attrib for x in arch_tag.findall("measurement")]): print("Couldn't find port utilization, skip: ", iform, file=sys.stderr) continue # skip if computed and measured TP don't match - if not [x.attrib['TP_ports'] == x.attrib['TP'] for x in arch_tag.findall('measurement')][ - 0 - ]: + if not [x.attrib["TP_ports"] == x.attrib["TP"] for x in arch_tag.findall("measurement")][0]: print( "Calculated TP from port utilization doesn't match TP, skip: ", iform, @@ -151,33 +149,31 @@ def extract_model(tree, arch, skip_mem=True): continue # skip if instruction contains memory operand if skip_mem and any( - [x.attrib['type'] == 'mem' for x in instruction_tag.findall('operand')] + [x.attrib["type"] == "mem" for x in instruction_tag.findall("operand")] ): print("Contains memory operand, skip: ", iform, file=sys.stderr) continue # We collect all measurement and IACA information and compare them later - for measurement_tag in arch_tag.iter('measurement'): - if 'TP_ports' in measurement_tag.attrib: - throughput = measurement_tag.attrib['TP_ports'] + for measurement_tag in arch_tag.iter("measurement"): + if "TP_ports" in measurement_tag.attrib: + throughput = measurement_tag.attrib["TP_ports"] else: throughput = ( - measurement_tag.attrib['TP'] if 'TP' in measurement_tag.attrib else None + measurement_tag.attrib["TP"] if "TP" in measurement_tag.attrib else None ) - uops = ( - int(measurement_tag.attrib['uops']) if 'uops' in measurement_tag.attrib else None - ) - if 'ports' in measurement_tag.attrib: + uops = int(measurement_tag.attrib["uops"]) if "uops" in measurement_tag.attrib else None + if "ports" in measurement_tag.attrib: port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib)) latencies = [ - int(l_tag.attrib['cycles']) - for l_tag in measurement_tag.iter('latency') - if 'cycles' in l_tag.attrib + int(l_tag.attrib["cycles"]) + for l_tag in measurement_tag.iter("latency") + if "cycles" in l_tag.attrib ] if len(latencies) == 0: latencies = [ - int(l_tag.attrib['max_cycles']) - for l_tag in measurement_tag.iter('latency') - if 'max_cycles' in l_tag.attrib + int(l_tag.attrib["max_cycles"]) + for l_tag in measurement_tag.iter("latency") + if "max_cycles" in l_tag.attrib ] if latencies[1:] != latencies[:-1]: print( @@ -193,9 +189,9 @@ def extract_model(tree, arch, skip_mem=True): # Ordered by IACA version (newest last) for iaca_tag in sorted( - arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version']) + arch_tag.iter("IACA"), key=lambda i: StrictVersion(i.attrib["version"]) ): - if 'ports' in iaca_tag.attrib: + if "ports" in iaca_tag.attrib: port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib)) # Check if all are equal @@ -208,26 +204,27 @@ def extract_model(tree, arch, skip_mem=True): continue # Adding Intel's 2D and 3D pipelines on Intel µarchs, without Ice Lake: - if arch.upper() in intel_archs and not arch.upper() in ['ICL']: - if any([p['class'] == 'memory' for p in parameters]): + if arch.upper() in intel_archs and not arch.upper() in ["ICL"]: + if any([p["class"] == "memory" for p in parameters]): # We have a memory parameter, if ports 2 & 3 are present, also add 2D & 3D # TODO remove port7 on 'hsw' onward and split entries depending on addressing mode port_23 = False port_4 = False for i, pp in enumerate(port_pressure): - if '2' in pp[1] and '3' in pp[1]: + if "2" in pp[1] and "3" in pp[1]: port_23 = True - if '4' in pp[1]: + if "4" in pp[1]: port_4 = True # Add (X, ['2D', '3D']) if load ports (2 & 3) are used, but not the store port (4) # X = 2 on SNB and IVB IFF used in combination with ymm register, otherwise X = 1 - if arch.upper() in ['SNB', 'IVB'] and \ - any([p['class'] == 'register' and p['name'] == 'ymm' for p in parameters]): + if arch.upper() in ["SNB", "IVB"] and any( + [p["class"] == "register" and p["name"] == "ymm" for p in parameters] + ): data_port_throughput = 2 else: data_port_throughput = 1 if port_23 and not port_4: - port_pressure.append((data_port_throughput, ['2D', '3D'])) + port_pressure.append((data_port_throughput, ["2D", "3D"])) # Add missing ports: for ports in [pp[1] for pp in port_pressure]: @@ -242,59 +239,55 @@ def extract_model(tree, arch, skip_mem=True): def rhs_comment(uncommented_string, comment): - max_length = max([len(l) for l in uncommented_string.split('\n')]) + max_length = max([len(l) for l in uncommented_string.split("\n")]) commented_string = "" - for l in uncommented_string.split('\n'): + for l in uncommented_string.split("\n"): commented_string += ("{:<" + str(max_length) + "} # {}\n").format(l, comment) return commented_string def architectures(tree): - return set([a.attrib['name'] for a in tree.findall('.//architecture')]) + return set([a.attrib["name"] for a in tree.findall(".//architecture")]) def main(): parser = argparse.ArgumentParser() - parser.add_argument('xml', help='path of instructions.xml from http://uops.info') + parser.add_argument("xml", help="path of instructions.xml from http://uops.info") parser.add_argument( - 'arch', - nargs='?', - help='architecture to extract, use IACA abbreviations (e.g., SNB). ' - 'if not given, all will be extracted and saved to file in CWD.', + "arch", + nargs="?", + help="architecture to extract, use IACA abbreviations (e.g., SNB). " + "if not given, all will be extracted and saved to file in CWD.", ) parser.add_argument( - '--mem', - dest='skip_mem', - action='store_false', - help='add instruction forms including memory addressing operands, which are ' - 'skipped by default' + "--mem", + dest="skip_mem", + action="store_false", + help="add instruction forms including memory addressing operands, which are " + "skipped by default", ) args = parser.parse_args() basename = os.path.basename(__file__) tree = ET.parse(args.xml) - print('# Available architectures:', ', '.join(architectures(tree))) + print("# Available architectures:", ", ".join(architectures(tree))) if args.arch: - print('# Chosen architecture: {}'.format(args.arch)) + print("# Chosen architecture: {}".format(args.arch)) model = extract_model(tree, args.arch, args.skip_mem) if model is not None: - print( - rhs_comment( - model.dump(), "uops.info import" - ) - ) + print(rhs_comment(model.dump(), "uops.info import")) else: for arch in architectures(tree): - print(arch, end='') + print(arch, end="") model = extract_model(tree, arch.lower(), args.skip_mem) if model: model_string = rhs_comment(model.dump(), basename + " " + arch) - with open('{}.yml'.format(arch.lower()), 'w') as f: + with open("{}.yml".format(arch.lower()), "w") as f: f.write(model_string) - print('.') + print(".") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/osaca/db_interface.py b/osaca/db_interface.py index 75898d3..5bc43ad 100755 --- a/osaca/db_interface.py +++ b/osaca/db_interface.py @@ -30,10 +30,10 @@ def sanity_check(arch: str, verbose=False, internet_check=False, output_file=sys """ # load arch machine model arch_mm = MachineModel(arch=arch) - data = arch_mm['instruction_forms'] + data = arch_mm["instruction_forms"] # load isa machine model isa = arch_mm.get_ISA() - isa_mm = MachineModel(arch='isa/{}'.format(isa)) + isa_mm = MachineModel(arch="isa/{}".format(isa)) num_of_instr = len(data) # check arch DB entries @@ -79,17 +79,17 @@ def import_benchmark_output(arch, bench_type, filepath, output=sys.stdout): :param output: output stream to dump, defaults to sys.stdout :type output: stream """ - supported_bench_outputs = ['ibench', 'asmbench'] + supported_bench_outputs = ["ibench", "asmbench"] assert os.path.exists(filepath) if bench_type not in supported_bench_outputs: - raise ValueError('Benchmark type is not supported.') - with open(filepath, 'r') as f: + raise ValueError("Benchmark type is not supported.") + with open(filepath, "r") as f: input_data = f.readlines() db_entries = None mm = MachineModel(arch) - if bench_type == 'ibench': + if bench_type == "ibench": db_entries = _get_ibench_output(input_data, mm.get_ISA()) - elif bench_type == 'asmbench': + elif bench_type == "asmbench": db_entries = _get_asmbench_output(input_data, mm.get_ISA()) # write entries to DB for entry in db_entries: @@ -122,34 +122,34 @@ def _get_asmbench_output(input_data, isa): """ db_entries = {} for i in range(0, len(input_data), 4): - if input_data[i + 3].strip() != '': - print('asmbench output not in the correct format! Format must be: ', file=sys.stderr) + if input_data[i + 3].strip() != "": + print("asmbench output not in the correct format! Format must be: ", file=sys.stderr) print( - '-------------\nMNEMONIC[-OP1[_OP2][...]]\nLatency: X cycles\n' - 'Throughput: Y cycles\n\n-------------', + "-------------\nMNEMONIC[-OP1[_OP2][...]]\nLatency: X cycles\n" + "Throughput: Y cycles\n\n-------------", file=sys.stderr, ) print( - 'Entry {} and all further entries won\'t be added.'.format((i / 4) + 1), + "Entry {} and all further entries won't be added.".format((i / 4) + 1), file=sys.stderr, ) break else: i_form = input_data[i].strip() - mnemonic = i_form.split('-')[0] - operands = i_form.split('-')[1].split('_') + mnemonic = i_form.split("-")[0] + operands = i_form.split("-")[1].split("_") operands = [_create_db_operand(op, isa) for op in operands] entry = { - 'name': mnemonic, - 'operands': operands, - 'throughput': _validate_measurement(float(input_data[i + 2].split()[1]), 'tp'), - 'latency': _validate_measurement(float(input_data[i + 1].split()[1]), 'lt'), - 'port_pressure': None, + "name": mnemonic, + "operands": operands, + "throughput": _validate_measurement(float(input_data[i + 2].split()[1]), "tp"), + "latency": _validate_measurement(float(input_data[i + 1].split()[1]), "lt"), + "port_pressure": None, } - if not entry['throughput'] or not entry['latency']: + if not entry["throughput"] or not entry["latency"]: warnings.warn( - 'Your measurement for {} looks suspicious'.format(i_form) - + ' and was not added. Please inspect your benchmark.' + "Your measurement for {} looks suspicious".format(i_form) + + " and was not added. Please inspect your benchmark." ) db_entries[i_form] = entry return db_entries @@ -159,37 +159,37 @@ def _get_ibench_output(input_data, isa): """Parse the standard output of ibench and add instructions to DB.""" db_entries = {} for line in input_data: - if 'Using frequency' in line or len(line) == 0: + if "Using frequency" in line or len(line) == 0: continue - instruction = line.split(':')[0] - key = '-'.join(instruction.split('-')[:2]) + instruction = line.split(":")[0] + key = "-".join(instruction.split("-")[:2]) if key in db_entries: # add only TP/LT value entry = db_entries[key] else: - mnemonic = instruction.split('-')[0] - operands = instruction.split('-')[1].split('_') + mnemonic = instruction.split("-")[0] + operands = instruction.split("-")[1].split("_") operands = [_create_db_operand(op, isa) for op in operands] entry = { - 'name': mnemonic, - 'operands': operands, - 'throughput': None, - 'latency': None, - 'port_pressure': None, + "name": mnemonic, + "operands": operands, + "throughput": None, + "latency": None, + "port_pressure": None, } - if 'TP' in instruction: - entry['throughput'] = _validate_measurement(float(line.split()[1]), 'tp') - if not entry['throughput']: + if "TP" in instruction: + entry["throughput"] = _validate_measurement(float(line.split()[1]), "tp") + if not entry["throughput"]: warnings.warn( - 'Your THROUGHPUT measurement for {} looks suspicious'.format(key) - + ' and was not added. Please inspect your benchmark.' + "Your THROUGHPUT measurement for {} looks suspicious".format(key) + + " and was not added. Please inspect your benchmark." ) - elif 'LT' in instruction: - entry['latency'] = _validate_measurement(float(line.split()[1]), 'lt') - if not entry['latency']: + elif "LT" in instruction: + entry["latency"] = _validate_measurement(float(line.split()[1]), "lt") + if not entry["latency"]: warnings.warn( - 'Your LATENCY measurement for {} looks suspicious'.format(key) - + ' and was not added. Please inspect your benchmark.' + "Your LATENCY measurement for {} looks suspicious".format(key) + + " and was not added. Please inspect your benchmark." ) db_entries[key] = entry return db_entries @@ -200,7 +200,7 @@ def _validate_measurement(measurement, mode): Check if latency has a maximum deviation of 0.05% and throughput is a reciprocal of a an integer number. """ - if mode == 'lt': + if mode == "lt": if ( math.floor(measurement) * 1.05 >= measurement or math.ceil(measurement) * 0.95 <= measurement @@ -208,7 +208,7 @@ def _validate_measurement(measurement, mode): # Value is probably correct, so round it to the estimated value return float(round(measurement)) # Check reciprocal only if it is a throughput value - elif mode == 'tp': + elif mode == "tp": reciprocals = [1 / x for x in range(1, 11)] for reci in reciprocals: if reci * 0.95 <= measurement <= reci * 1.05: @@ -221,56 +221,56 @@ def _validate_measurement(measurement, mode): def _create_db_operand(operand, isa): """Get DB operand by input string and ISA.""" - if isa == 'aarch64': + if isa == "aarch64": return _create_db_operand_aarch64(operand) - elif isa == 'x86': + elif isa == "x86": return _create_db_operand_x86(operand) def _create_db_operand_aarch64(operand): """Get DB operand for AArch64 by operand string.""" - if operand == 'i': - return {'class': 'immediate', 'imd': 'int'} - elif operand in 'wxbhsdq': - return {'class': 'register', 'prefix': operand} - elif operand.startswith('v'): + if operand == "i": + return {"class": "immediate", "imd": "int"} + elif operand in "wxbhsdq": + return {"class": "register", "prefix": operand} + elif operand.startswith("v"): return { - 'class': 'register', - 'prefix': 'v', - 'shape': operand[1:2] if operand[1:2] != '' else 'd', + "class": "register", + "prefix": "v", + "shape": operand[1:2] if operand[1:2] != "" else "d", } - elif operand.startswith('m'): + elif operand.startswith("m"): return { - 'class': 'memory', - 'base': 'x' if 'b' in operand else None, - 'offset': 'imd' if 'o' in operand else None, - 'index': 'gpr' if 'i' in operand else None, - 'scale': 8 if 's' in operand else 1, - 'pre-indexed': True if 'r' in operand else False, - 'post-indexed': True if 'p' in operand else False, + "class": "memory", + "base": "x" if "b" in operand else None, + "offset": "imd" if "o" in operand else None, + "index": "gpr" if "i" in operand else None, + "scale": 8 if "s" in operand else 1, + "pre-indexed": True if "r" in operand else False, + "post-indexed": True if "p" in operand else False, } else: - raise ValueError('Parameter {} is not a valid operand code'.format(operand)) + raise ValueError("Parameter {} is not a valid operand code".format(operand)) def _create_db_operand_x86(operand): """Get DB operand for AArch64 by operand string.""" - if operand == 'r': - return {'class': 'register', 'name': 'gpr'} - elif operand in 'xyz': - return {'class': 'register', 'name': operand + 'mm'} - elif operand == 'i': - return {'class': 'immediate', 'imd': 'int'} - elif operand.startswith('m'): + if operand == "r": + return {"class": "register", "name": "gpr"} + elif operand in "xyz": + return {"class": "register", "name": operand + "mm"} + elif operand == "i": + return {"class": "immediate", "imd": "int"} + elif operand.startswith("m"): return { - 'class': 'memory', - 'base': 'gpr' if 'b' in operand else None, - 'offset': 'imd' if 'o' in operand else None, - 'index': 'gpr' if 'i' in operand else None, - 'scale': 8 if 's' in operand else 1, + "class": "memory", + "base": "gpr" if "b" in operand else None, + "offset": "imd" if "o" in operand else None, + "index": "gpr" if "i" in operand else None, + "scale": 8 if "s" in operand else 1, } else: - raise ValueError('Parameter {} is not a valid operand code'.format(operand)) + raise ValueError("Parameter {} is not a valid operand code".format(operand)) ######################## @@ -286,14 +286,14 @@ def _scrape_from_felixcloutier(mnemonic): from bs4 import BeautifulSoup except ImportError: print( - 'Module BeautifulSoup not installed. Fetching instruction form information ' - 'online requires BeautifulSoup.\nUse \'pip install bs4\' for installation.', + "Module BeautifulSoup not installed. Fetching instruction form information " + "online requires BeautifulSoup.\nUse 'pip install bs4' for installation.", file=sys.stderr, ) sys.exit(1) - index = 'https://www.felixcloutier.com/x86/index.html' - base_url = 'https://www.felixcloutier.com/x86/' + index = "https://www.felixcloutier.com/x86/index.html" + base_url = "https://www.felixcloutier.com/x86/" url = base_url + mnemonic.lower() suspicious = True @@ -303,8 +303,8 @@ def _scrape_from_felixcloutier(mnemonic): r = requests.get(url=url) if r.status_code == 200: # Found result - operand_enc = BeautifulSoup(r.text, 'html.parser').find( - 'h2', attrs={'id': 'instruction-operand-encoding'} + operand_enc = BeautifulSoup(r.text, "html.parser").find( + "h2", attrs={"id": "instruction-operand-encoding"} ) if operand_enc: # operand encoding found, otherwise, no need to mark as suspicous @@ -312,37 +312,35 @@ def _scrape_from_felixcloutier(mnemonic): operands = _get_src_dst_from_table(table) elif r.status_code == 404: # Check for alternative href - index = BeautifulSoup(requests.get(url=index).text, 'html.parser') - alternatives = [ref for ref in index.findAll('a') if ref.text == mnemonic.upper()] + index = BeautifulSoup(requests.get(url=index).text, "html.parser") + alternatives = [ref for ref in index.findAll("a") if ref.text == mnemonic.upper()] if len(alternatives) > 0: # alternative(s) found, take first one - url = base_url + alternatives[0].attrs['href'][2:] - operand_enc = BeautifulSoup(requests.get(url=url).text, 'html.parser').find( - 'h2', attrs={'id': 'instruction-operand-encoding'} + url = base_url + alternatives[0].attrs["href"][2:] + operand_enc = BeautifulSoup(requests.get(url=url).text, "html.parser").find( + "h2", attrs={"id": "instruction-operand-encoding"} ) if operand_enc: # operand encoding found, otherwise, no need to mark as suspicous - table = ( - operand_enc.findNextSibling() - ) + table = operand_enc.findNextSibling() operands = _get_src_dst_from_table(table) if operands: # Found src/dst assignment for NUM_OPERANDS - if not any(['r' in x and 'w' in x for x in operands]): + if not any(["r" in x and "w" in x for x in operands]): suspicious = False - return (suspicious, ' '.join(operands)) + return (suspicious, " ".join(operands)) def _get_src_dst_from_table(table, num_operands=2): """Prettify bs4 table object to string for user""" # Parse table - header = [''.join(x.string.lower().split()) for x in table.find('tr').findAll('td')] - data = table.findAll('tr')[1:] + header = ["".join(x.string.lower().split()) for x in table.find("tr").findAll("td")] + data = table.findAll("tr")[1:] data_dict = OrderedDict() for i, row in enumerate(data): data_dict[i] = {} - for j, col in enumerate(row.findAll('td')): - if col.string != 'NA': + for j, col in enumerate(row.findAll("td")): + if col.string != "NA": data_dict[i][header[j]] = col.string # Get only the instruction forms with 2 operands num_ops = [_get_number_of_operands(row) for _, row in data_dict.items()] @@ -350,12 +348,12 @@ def _get_src_dst_from_table(table, num_operands=2): row = data_dict[num_ops.index(num_operands)] reads_writes = [] for i in range(1, num_operands + 1): - m = re.search(r'(\([^\(\)]+\))', row['operand{}'.format(i)]) + m = re.search(r"(\([^\(\)]+\))", row["operand{}".format(i)]) if not m: # no parentheses (probably immediate operand), assume READ - reads_writes.append('(r)') + reads_writes.append("(r)") continue - reads_writes.append(''.join(m.group(0).split())) + reads_writes.append("".join(m.group(0).split())) # reverse reads_writes for AT&T syntax reads_writes.reverse() return reads_writes @@ -366,7 +364,7 @@ def _get_number_of_operands(data_dict_row): """Return the number of `Operand [X]` attributes in row""" num = 0 for i in range(1, 5): - if 'operand{}'.format(i) in [''.join(x.split()).lower() for x in data_dict_row]: + if "operand{}".format(i) in ["".join(x.split()).lower() for x in data_dict_row]: num += 1 return num @@ -374,12 +372,12 @@ def _get_number_of_operands(data_dict_row): def _check_sanity_arch_db(arch_mm, isa_mm, internet_check=True): """Do sanity check for ArchDB by given ISA.""" # prefixes of instruction forms which we assume to have non-default operands - suspicious_prefixes_x86 = ['vfm', 'fm'] - suspicious_prefixes_arm = ['fml', 'ldp', 'stp', 'str'] + suspicious_prefixes_x86 = ["vfm", "fm"] + suspicious_prefixes_arm = ["fml", "ldp", "stp", "str"] # already known to be default-operand instruction forms with 2 operands - if arch_mm.get_ISA().lower() == 'aarch64': + if arch_mm.get_ISA().lower() == "aarch64": suspicious_prefixes = suspicious_prefixes_arm - if arch_mm.get_ISA().lower() == 'x86': + if arch_mm.get_ISA().lower() == "x86": suspicious_prefixes = suspicious_prefixes_x86 # returned lists @@ -391,57 +389,56 @@ def _check_sanity_arch_db(arch_mm, isa_mm, internet_check=True): duplicate_strings = [] bad_operand = [] - for instr_form in arch_mm['instruction_forms']: + for instr_form in arch_mm["instruction_forms"]: # check value in DB entry - if instr_form['throughput'] is None: + if instr_form["throughput"] is None: missing_throughput.append(instr_form) - if instr_form['latency'] is None: + if instr_form["latency"] is None: missing_latency.append(instr_form) - if instr_form['port_pressure'] is None: + if instr_form["port_pressure"] is None: missing_port_pressure.append(instr_form) # check entry against ISA DB for prefix in suspicious_prefixes: - if instr_form['name'].lower().startswith(prefix): + if instr_form["name"].lower().startswith(prefix): # check if instruction in ISA DB - if isa_mm.get_instruction(instr_form['name'], instr_form['operands']) is None: + if isa_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None: # if not, mark them as suspicious and print it on the screen suspicious_instructions.append(instr_form) # instr forms with less than 3 operands might need an ISA DB entry due to src_reg operands if ( - len(instr_form['operands']) < 3 - and len(instr_form['operands']) > 1 - and 'mov' not in instr_form['name'].lower() - and not instr_form['name'].lower().startswith('j') + len(instr_form["operands"]) < 3 + and len(instr_form["operands"]) > 1 + and "mov" not in instr_form["name"].lower() + and not instr_form["name"].lower().startswith("j") and instr_form not in suspicious_instructions - and isa_mm.get_instruction(instr_form['name'], instr_form['operands']) is None + and isa_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None ): # validate with data from internet if connected flag is set if internet_check: - is_susp, info_string = _scrape_from_felixcloutier(instr_form['name']) + is_susp, info_string = _scrape_from_felixcloutier(instr_form["name"]) if is_susp: - instr_form['note'] = info_string + instr_form["note"] = info_string suspicious_instructions.append(instr_form) else: suspicious_instructions.append(instr_form) # check for duplicates in DB - if arch_mm._check_for_duplicate(instr_form['name'], instr_form['operands']): + if arch_mm._check_for_duplicate(instr_form["name"], instr_form["operands"]): duplicate_instr_arch.append(instr_form) - + # Check operands - for operand in instr_form['operands']: - if operand['class'] == 'register' and not ( - 'name' in operand or - 'prefix' in operand): + for operand in instr_form["operands"]: + if operand["class"] == "register" and not ("name" in operand or "prefix" in operand): # Missing 'name' key bad_operand.append(instr_form) - elif operand['class'] == 'memory' and ( - 'base' not in operand or - 'offset' not in operand or - 'index' not in operand or - 'scale' not in operand): + elif operand["class"] == "memory" and ( + "base" not in operand + or "offset" not in operand + or "index" not in operand + or "scale" not in operand + ): # Missing at least one key necessary for memory operands bad_operand.append(instr_form) - elif operand['class'] == 'immediate' and 'imd' not in operand: + elif operand["class"] == "immediate" and "imd" not in operand: # Missing 'imd' key bad_operand.append(instr_form) # every entry exists twice --> uniquify @@ -468,12 +465,12 @@ def _check_sanity_isa_db(arch_mm, isa_mm): duplicate_instr_isa = [] only_in_isa = [] - for instr_form in isa_mm['instruction_forms']: + for instr_form in isa_mm["instruction_forms"]: # check if instr is missing in arch DB - if arch_mm.get_instruction(instr_form['name'], instr_form['operands']) is None: + if arch_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None: only_in_isa.append(instr_form) # check for duplicates - if isa_mm._check_for_duplicate(instr_form['name'], instr_form['operands']): + if isa_mm._check_for_duplicate(instr_form["name"], instr_form["operands"]): duplicate_instr_isa.append(instr_form) # every entry exists twice --> uniquify tmp_list = [] @@ -486,82 +483,102 @@ def _check_sanity_isa_db(arch_mm, isa_mm): return duplicate_instr_isa, only_in_isa -def _get_sanity_report(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, - bad_operands, verbose=False, colors=False): +def _get_sanity_report( + total, + m_tp, + m_l, + m_pp, + suspic_instr, + dup_arch, + dup_isa, + only_isa, + bad_operands, + verbose=False, + colors=False, +): """Get sanity summary report.""" - s = '' + s = "" # non-verbose summary - s += 'SUMMARY\n----------------------\n' - s += '{}% ({}/{}) of instruction forms have no throughput value.\n'.format( + s += "SUMMARY\n----------------------\n" + s += "{}% ({}/{}) of instruction forms have no throughput value.\n".format( round(100 * len(m_tp) / total), len(m_tp), total ) - s += '{}% ({}/{}) of instruction forms have no latency value.\n'.format( + s += "{}% ({}/{}) of instruction forms have no latency value.\n".format( round(100 * len(m_l) / total), len(m_l), total ) - s += '{}% ({}/{}) of instruction forms have no port pressure assignment.\n'.format( + s += "{}% ({}/{}) of instruction forms have no port pressure assignment.\n".format( round(100 * len(m_pp) / total), len(m_pp), total ) - s += '{}% ({}/{}) of instruction forms might miss an ISA DB entry.\n'.format( + s += "{}% ({}/{}) of instruction forms might miss an ISA DB entry.\n".format( round(100 * len(suspic_instr) / total), len(suspic_instr), total ) - s += '{} duplicate instruction forms in uarch DB.\n'.format(len(dup_arch)) - s += '{} duplicate instruction forms in ISA DB.\n'.format(len(dup_isa)) + s += "{} duplicate instruction forms in uarch DB.\n".format(len(dup_arch)) + s += "{} duplicate instruction forms in ISA DB.\n".format(len(dup_isa)) s += ( - '{} instruction forms in ISA DB are not referenced by instruction '.format(len(only_isa)) - + 'forms in uarch DB.\n' + "{} instruction forms in ISA DB are not referenced by instruction ".format(len(only_isa)) + + "forms in uarch DB.\n" ) - s += '{} bad operands found in uarch DB\n'.format(len(bad_operands)) - s += '----------------------\n' + s += "{} bad operands found in uarch DB\n".format(len(bad_operands)) + s += "----------------------\n" # verbose version if verbose: s += _get_sanity_report_verbose( - total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, bad_operands, - colors=colors + total, + m_tp, + m_l, + m_pp, + suspic_instr, + dup_arch, + dup_isa, + only_isa, + bad_operands, + colors=colors, ) return s -def _get_sanity_report_verbose(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, - bad_operands, colors=False): +def _get_sanity_report_verbose( + total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, bad_operands, colors=False +): """Get the verbose part of the sanity report with all missing instruction forms.""" - BRIGHT_CYAN = '\033[1;36;1m' if colors else '' - BRIGHT_BLUE = '\033[1;34;1m' if colors else '' - BRIGHT_RED = '\033[1;31;1m' if colors else '' - BRIGHT_MAGENTA = '\033[1;35;1m' if colors else '' - BRIGHT_YELLOW = '\033[1;33;1m' if colors else '' - CYAN = '\033[36m' if colors else '' - YELLOW = '\033[33m' if colors else '' - WHITE = '\033[0m' if colors else '' + BRIGHT_CYAN = "\033[1;36;1m" if colors else "" + BRIGHT_BLUE = "\033[1;34;1m" if colors else "" + BRIGHT_RED = "\033[1;31;1m" if colors else "" + BRIGHT_MAGENTA = "\033[1;35;1m" if colors else "" + BRIGHT_YELLOW = "\033[1;33;1m" if colors else "" + CYAN = "\033[36m" if colors else "" + YELLOW = "\033[33m" if colors else "" + WHITE = "\033[0m" if colors else "" - s = 'Instruction forms without throughput value:\n' if m_tp else '' - for instr_form in sorted(m_tp, key=lambda i: i['name']): - s += '{}{}{}\n'.format(BRIGHT_BLUE, _get_full_instruction_name(instr_form), WHITE) - s += 'Instruction forms without latency value:\n' if m_l else '' - for instr_form in sorted(m_l, key=lambda i: i['name']): - s += '{}{}{}\n'.format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE) - s += 'Instruction forms without port pressure assignment:\n' if m_pp else '' - for instr_form in sorted(m_pp, key=lambda i: i['name']): - s += '{}{}{}\n'.format(BRIGHT_MAGENTA, _get_full_instruction_name(instr_form), WHITE) - s += 'Instruction forms which might miss an ISA DB entry:\n' if suspic_instr else '' - for instr_form in sorted(suspic_instr, key=lambda i: i['name']): - s += '{}{}{}{}\n'.format( + s = "Instruction forms without throughput value:\n" if m_tp else "" + for instr_form in sorted(m_tp, key=lambda i: i["name"]): + s += "{}{}{}\n".format(BRIGHT_BLUE, _get_full_instruction_name(instr_form), WHITE) + s += "Instruction forms without latency value:\n" if m_l else "" + for instr_form in sorted(m_l, key=lambda i: i["name"]): + s += "{}{}{}\n".format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE) + s += "Instruction forms without port pressure assignment:\n" if m_pp else "" + for instr_form in sorted(m_pp, key=lambda i: i["name"]): + s += "{}{}{}\n".format(BRIGHT_MAGENTA, _get_full_instruction_name(instr_form), WHITE) + s += "Instruction forms which might miss an ISA DB entry:\n" if suspic_instr else "" + for instr_form in sorted(suspic_instr, key=lambda i: i["name"]): + s += "{}{}{}{}\n".format( BRIGHT_CYAN, _get_full_instruction_name(instr_form), - ' -- ' + instr_form['note'] if 'note' in instr_form else '', + " -- " + instr_form["note"] if "note" in instr_form else "", WHITE, ) - s += 'Duplicate instruction forms in uarch DB:\n' if dup_arch else '' - for instr_form in sorted(dup_arch, key=lambda i: i['name']): - s += '{}{}{}\n'.format(YELLOW, _get_full_instruction_name(instr_form), WHITE) - s += 'Duplicate instruction forms in ISA DB:\n' if dup_isa else '' - for instr_form in sorted(dup_isa, key=lambda i: i['name']): - s += '{}{}{}\n'.format(BRIGHT_YELLOW, _get_full_instruction_name(instr_form), WHITE) - s += 'Instruction forms existing in ISA DB but not in uarch DB:\n' if only_isa else '' - for instr_form in sorted(only_isa, key=lambda i: i['name']): - s += '{}{}{}\n'.format(CYAN, _get_full_instruction_name(instr_form), WHITE) - s += '{} bad operands found in uarch DB:\n'.format(len(bad_operands)) if bad_operands else '' - for instr_form in sorted(bad_operands, key=lambda i: i['name']): - s += '{}{}{}\n'.format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE) + s += "Duplicate instruction forms in uarch DB:\n" if dup_arch else "" + for instr_form in sorted(dup_arch, key=lambda i: i["name"]): + s += "{}{}{}\n".format(YELLOW, _get_full_instruction_name(instr_form), WHITE) + s += "Duplicate instruction forms in ISA DB:\n" if dup_isa else "" + for instr_form in sorted(dup_isa, key=lambda i: i["name"]): + s += "{}{}{}\n".format(BRIGHT_YELLOW, _get_full_instruction_name(instr_form), WHITE) + s += "Instruction forms existing in ISA DB but not in uarch DB:\n" if only_isa else "" + for instr_form in sorted(only_isa, key=lambda i: i["name"]): + s += "{}{}{}\n".format(CYAN, _get_full_instruction_name(instr_form), WHITE) + s += "{} bad operands found in uarch DB:\n".format(len(bad_operands)) if bad_operands else "" + for instr_form in sorted(bad_operands, key=lambda i: i["name"]): + s += "{}{}{}\n".format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE) return s @@ -573,18 +590,18 @@ def _get_sanity_report_verbose(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, d def _get_full_instruction_name(instruction_form): """Get full instruction form name/identifier string out of given instruction form.""" operands = [] - for op in instruction_form['operands']: + for op in instruction_form["operands"]: op_attrs = [ - y + ':' + str(op[y]) - for y in list(filter(lambda x: True if x != 'class' else False, op)) + y + ":" + str(op[y]) + for y in list(filter(lambda x: True if x != "class" else False, op)) ] - operands.append('{}({})'.format(op['class'], ','.join(op_attrs))) - return '{} {}'.format(instruction_form['name'], ','.join(operands)) + operands.append("{}({})".format(op["class"], ",".join(op_attrs))) + return "{} {}".format(instruction_form["name"], ",".join(operands)) def __represent_none(self, data): """Get YAML None representation.""" - return self.represent_scalar(u'tag:yaml.org,2002:null', u'~') + return self.represent_scalar(u"tag:yaml.org,2002:null", u"~") def _create_yaml_object(): @@ -598,17 +615,17 @@ def __dump_data_to_yaml(filepath, data): """Dump data to YAML file at given filepath.""" # first add 'normal' meta data in the right order (no ordered dict yet) meta_data = dict(data) - del meta_data['instruction_forms'] - del meta_data['port_model_scheme'] - with open(filepath, 'w') as f: + del meta_data["instruction_forms"] + del meta_data["port_model_scheme"] + with open(filepath, "w") as f: ruamel.yaml.dump(meta_data, f, allow_unicode=True) - with open(filepath, 'a') as f: + with open(filepath, "a") as f: # now add port model scheme in |-scheme for better readability ruamel.yaml.dump( - {'port_model_scheme': data['port_model_scheme']}, + {"port_model_scheme": data["port_model_scheme"]}, f, allow_unicode=True, - default_style='|', + default_style="|", ) # finally, add instruction forms - ruamel.yaml.dump({'instruction_forms': data['instruction_forms']}, f, allow_unicode=True) + ruamel.yaml.dump({"instruction_forms": data["instruction_forms"]}, f, allow_unicode=True) diff --git a/osaca/frontend.py b/osaca/frontend.py index 07baaef..4a7e5ae 100755 --- a/osaca/frontend.py +++ b/osaca/frontend.py @@ -9,20 +9,19 @@ from datetime import datetime as dt from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel + def _get_version(*file_paths): """Searches for a version attribute in the given file(s)""" - with io.open( - os.path.join(os.path.dirname(__file__), *file_paths), encoding='utf-8' - ) as fp: + with io.open(os.path.join(os.path.dirname(__file__), *file_paths), encoding="utf-8") as fp: version_file = fp.read() version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) if version_match: return version_match.group(1) - return '?.?.?' + return "?.?.?" class Frontend(object): - def __init__(self, filename='', arch=None, path_to_yaml=None): + def __init__(self, filename="", arch=None, path_to_yaml=None): """ Constructor method. @@ -35,9 +34,9 @@ class Frontend(object): """ self._filename = filename if not arch and not path_to_yaml: - raise ValueError('Either arch or path_to_yaml required.') + raise ValueError("Either arch or path_to_yaml required.") if arch and path_to_yaml: - raise ValueError('Only one of arch and path_to_yaml is allowed.') + raise ValueError("Only one of arch and path_to_yaml is allowed.") self._arch = arch if arch: self._arch = arch.lower() @@ -54,7 +53,7 @@ class Frontend(object): :type instruction_form: `dict` :returns: `True` if comment line, `False` otherwise """ - return instruction_form['comment'] is not None and instruction_form['instruction'] is None + return instruction_form["comment"] is not None and instruction_form["instruction"] is None def throughput_analysis(self, kernel, show_lineno=False, show_cmnts=True): """ @@ -67,40 +66,40 @@ class Frontend(object): :param show_cmnts: flag for showing comment-only lines in kernel, defaults to `True` :type show_cmnts: bool, optional """ - lineno_filler = ' ' if show_lineno else '' + lineno_filler = " " if show_lineno else "" port_len = self._get_max_port_len(kernel) - separator = '-' * sum([x + 3 for x in port_len]) + '-' - separator += '--' + len(str(kernel[-1]['line_number'])) * '-' if show_lineno else '' - col_sep = '|' + separator = "-" * sum([x + 3 for x in port_len]) + "-" + separator += "--" + len(str(kernel[-1]["line_number"])) * "-" if show_lineno else "" + col_sep = "|" sep_list = self._get_separator_list(col_sep) - headline = 'Port pressure in cycles' - headline_str = '{{:^{}}}'.format(len(separator)) + headline = "Port pressure in cycles" + headline_str = "{{:^{}}}".format(len(separator)) - s = '\n\nThroughput Analysis Report\n--------------------------\n' - s += headline_str.format(headline) + '\n' - s += lineno_filler + self._get_port_number_line(port_len) + '\n' - s += separator + '\n' + s = "\n\nThroughput Analysis Report\n--------------------------\n" + s += headline_str.format(headline) + "\n" + s += lineno_filler + self._get_port_number_line(port_len) + "\n" + s += separator + "\n" for instruction_form in kernel: - line = '{:4d} {} {} {}'.format( - instruction_form['line_number'], + line = "{:4d} {} {} {}".format( + instruction_form["line_number"], self._get_port_pressure( - instruction_form['port_pressure'], port_len, separator=sep_list + instruction_form["port_pressure"], port_len, separator=sep_list ), - self._get_flag_symbols(instruction_form['flags']) - if instruction_form['instruction'] is not None - else ' ', - instruction_form['line'].strip().replace('\t', ' '), + self._get_flag_symbols(instruction_form["flags"]) + if instruction_form["instruction"] is not None + else " ", + instruction_form["line"].strip().replace("\t", " "), ) line = line if show_lineno else col_sep + col_sep.join(line.split(col_sep)[1:]) if show_cmnts is False and self._is_comment(instruction_form): continue - s += line + '\n' - s += '\n' + s += line + "\n" + s += "\n" tp_sum = ArchSemantics.get_throughput_sum(kernel) - s += lineno_filler + self._get_port_pressure(tp_sum, port_len, separator=' ') + '\n' + s += lineno_filler + self._get_port_pressure(tp_sum, port_len, separator=" ") + "\n" return s - def latency_analysis(self, cp_kernel, separator='|'): + def latency_analysis(self, cp_kernel, separator="|"): """ Build a list-based CP analysis report. @@ -109,29 +108,29 @@ class Frontend(object): :separator: separator symbol for the columns, defaults to '|' :type separator: str, optional """ - s = '\n\nLatency Analysis Report\n-----------------------\n' + s = "\n\nLatency Analysis Report\n-----------------------\n" for instruction_form in cp_kernel: s += ( - '{:4d} {} {:4.1f} {}{}{} {}'.format( - instruction_form['line_number'], + "{:4d} {} {:4.1f} {}{}{} {}".format( + instruction_form["line_number"], separator, - instruction_form['latency_cp'], + instruction_form["latency_cp"], separator, - 'X' if INSTR_FLAGS.LT_UNKWN in instruction_form['flags'] else ' ', + "X" if INSTR_FLAGS.LT_UNKWN in instruction_form["flags"] else " ", separator, - instruction_form['line'], + instruction_form["line"], ) - ) + '\n' + ) + "\n" s += ( - '\n{:4} {} {:4.1f}'.format( - ' ' * max([len(str(instr_form['line_number'])) for instr_form in cp_kernel]), - ' ' * len(separator), - sum([instr_form['latency_cp'] for instr_form in cp_kernel]), + "\n{:4} {} {:4.1f}".format( + " " * max([len(str(instr_form["line_number"])) for instr_form in cp_kernel]), + " " * len(separator), + sum([instr_form["latency_cp"] for instr_form in cp_kernel]), ) - ) + '\n' + ) + "\n" return s - def loopcarried_dependencies(self, dep_dict, separator='|'): + def loopcarried_dependencies(self, dep_dict, separator="|"): """ Print a list-based LCD analysis to the terminal. @@ -141,23 +140,31 @@ class Frontend(object): :type separator: str, optional """ s = ( - '\n\nLoop-Carried Dependencies Analysis Report\n' - + '-----------------------------------------\n' + "\n\nLoop-Carried Dependencies Analysis Report\n" + + "-----------------------------------------\n" ) # TODO find a way to overcome padding for different tab-lengths for dep in dep_dict: - s += '{:4d} {} {:4.1f} {} {:36}{} {}\n'.format( + s += "{:4d} {} {:4.1f} {} {:36}{} {}\n".format( dep, separator, - sum([instr_form['latency_lcd'] for instr_form in dep_dict[dep]['dependencies']]), + sum([instr_form["latency_lcd"] for instr_form in dep_dict[dep]["dependencies"]]), separator, - dep_dict[dep]['root']['line'].strip(), + dep_dict[dep]["root"]["line"].strip(), separator, - [node['line_number'] for node in dep_dict[dep]['dependencies']], + [node["line_number"] for node in dep_dict[dep]["dependencies"]], ) return s - def full_analysis(self, kernel, kernel_dg: KernelDG, ignore_unknown=False, arch_warning=False, length_warning=False, verbose=False): + def full_analysis( + self, + kernel, + kernel_dg: KernelDG, + ignore_unknown=False, + arch_warning=False, + length_warning=False, + verbose=False, + ): """ Build the full analysis report including header, the symbol map, the combined TP/CP/LCD view and the list based LCD view. @@ -208,71 +215,71 @@ class Frontend(object): :param show_cmnts: flag for showing comment-only lines in kernel, defaults to `True` :type show_cmnts: bool, optional """ - s = '\n\nCombined Analysis Report\n------------------------\n' - lineno_filler = ' ' + s = "\n\nCombined Analysis Report\n------------------------\n" + lineno_filler = " " port_len = self._get_max_port_len(kernel) # Separator for ports - separator = '-' * sum([x + 3 for x in port_len]) + '-' + separator = "-" * sum([x + 3 for x in port_len]) + "-" # ... for line numbers - separator += '--' + len(str(kernel[-1]['line_number'])) * '-' - col_sep = '|' + separator += "--" + len(str(kernel[-1]["line_number"])) * "-" + col_sep = "|" # for LCD/CP column - separator += '-' * (2 * 6 + len(col_sep)) + '-' * len(col_sep) + separator += "-" * (2 * 6 + len(col_sep)) + "-" * len(col_sep) sep_list = self._get_separator_list(col_sep) - headline = 'Port pressure in cycles' - headline_str = '{{:^{}}}'.format(len(separator)) + headline = "Port pressure in cycles" + headline_str = "{{:^{}}}".format(len(separator)) # Prepare CP/LCD variable - cp_lines = [x['line_number'] for x in cp_kernel] + cp_lines = [x["line_number"] for x in cp_kernel] sums = {} for dep in dep_dict: sums[dep] = sum( - [instr_form['latency_lcd'] for instr_form in dep_dict[dep]['dependencies']] + [instr_form["latency_lcd"] for instr_form in dep_dict[dep]["dependencies"]] ) lcd_sum = max(sums.values()) if len(sums) > 0 else 0.0 lcd_lines = [] if len(dep_dict) > 0: longest_lcd = [line_no for line_no in sums if sums[line_no] == lcd_sum][0] - lcd_lines = [d['line_number'] for d in dep_dict[longest_lcd]['dependencies']] + lcd_lines = [d["line_number"] for d in dep_dict[longest_lcd]["dependencies"]] - s += headline_str.format(headline) + '\n' + s += headline_str.format(headline) + "\n" s += ( ( lineno_filler + self._get_port_number_line(port_len, separator=col_sep) - + '{}{:^6}{}{:^6}{}'.format(col_sep, 'CP', col_sep, 'LCD', col_sep) + + "{}{:^6}{}{:^6}{}".format(col_sep, "CP", col_sep, "LCD", col_sep) ) - + '\n' + + "\n" + separator - + '\n' + + "\n" ) for instruction_form in kernel: if show_cmnts is False and self._is_comment(instruction_form): continue - line_number = instruction_form['line_number'] - used_ports = [list(uops[1]) for uops in instruction_form['port_uops']] + line_number = instruction_form["line_number"] + used_ports = [list(uops[1]) for uops in instruction_form["port_uops"]] used_ports = list(set([p for uops_ports in used_ports for p in uops_ports])) - s += '{:4d} {}{} {} {}\n'.format( + s += "{:4d} {}{} {} {}\n".format( line_number, self._get_port_pressure( - instruction_form['port_pressure'], port_len, used_ports, sep_list + instruction_form["port_pressure"], port_len, used_ports, sep_list ), self._get_lcd_cp_ports( - instruction_form['line_number'], + instruction_form["line_number"], cp_kernel if line_number in cp_lines else None, dep_dict[longest_lcd] if line_number in lcd_lines else None, ), - self._get_flag_symbols(instruction_form['flags']) - if instruction_form['instruction'] is not None - else ' ', - instruction_form['line'].strip().replace('\t', ' '), + self._get_flag_symbols(instruction_form["flags"]) + if instruction_form["instruction"] is not None + else " ", + instruction_form["line"].strip().replace("\t", " "), ) - s += '\n' + s += "\n" # check for unknown instructions and throw warning if called without --ignore-unknown if not ignore_unknown and INSTR_FLAGS.TP_UNKWN in [ - flag for instr in kernel for flag in instr['flags'] + flag for instr in kernel for flag in instr["flags"] ]: num_missing = len( - [instr['flags'] for instr in kernel if INSTR_FLAGS.TP_UNKWN in instr['flags']] + [instr["flags"] for instr in kernel if INSTR_FLAGS.TP_UNKWN in instr["flags"]] ) s += self._missing_instruction_error(num_missing) else: @@ -280,12 +287,12 @@ class Frontend(object): tp_sum = ArchSemantics.get_throughput_sum(kernel) # if ALL instructions are unknown, take a line of 0s if not tp_sum: - tp_sum = kernel[0]['port_pressure'] - cp_sum = sum([x['latency_cp'] for x in cp_kernel]) + tp_sum = kernel[0]["port_pressure"] + cp_sum = sum([x["latency_cp"] for x in cp_kernel]) s += ( lineno_filler - + self._get_port_pressure(tp_sum, port_len, separator=' ') - + ' {:^6} {:^6}\n'.format(cp_sum, lcd_sum) + + self._get_port_pressure(tp_sum, port_len, separator=" ") + + " {:^6} {:^6}\n".format(cp_sum, lcd_sum) ) return s @@ -296,43 +303,42 @@ class Frontend(object): def _missing_instruction_error(self, amount): """Returns the warning for if any instruction form in the analysis is missing.""" s = ( - '------------------ WARNING: The performance data for {} instructions is missing.' - '------------------\n' - ' No final analysis is given. If you want to ignore this\n' - ' warning and run the analysis anyway, start osaca with\n' - ' --ignore-unknown flag.\n' - '--------------------------------------------------------------------------------' - '----------------{}\n' - ).format(amount, '-' * len(str(amount))) + "------------------ WARNING: The performance data for {} instructions is missing." + "------------------\n" + " No final analysis is given. If you want to ignore this\n" + " warning and run the analysis anyway, start osaca with\n" + " --ignore-unknown flag.\n" + "--------------------------------------------------------------------------------" + "----------------{}\n" + ).format(amount, "-" * len(str(amount))) return s def _user_warnings(self, arch_warning, length_warning): """Returns warning texts for giving the user more insight in what he is doing.""" arch_text = ( - 'WARNING: No micro-architecture was specified and a default uarch was used.\n' - ' Specify the uarch with --arch. See --help for more information.\n' + "WARNING: No micro-architecture was specified and a default uarch was used.\n" + " Specify the uarch with --arch. See --help for more information.\n" ) length_text = ( - 'WARNING: You are analyzing a large amount of instruction forms. Analysis ' - 'across loops/block boundaries often do not make much sense.\n' - ' Specify the kernel length with --length. See --help for more ' - 'information.\n' - ' If this is intentional, you can safely ignore this message.\n' + "WARNING: You are analyzing a large amount of instruction forms. Analysis " + "across loops/block boundaries often do not make much sense.\n" + " Specify the kernel length with --length. See --help for more " + "information.\n" + " If this is intentional, you can safely ignore this message.\n" ) - warnings = '' - warnings += arch_text if arch_warning else '' - warnings += length_text if length_warning else '' - warnings += '\n' + warnings = "" + warnings += arch_text if arch_warning else "" + warnings += length_text if length_warning else "" + warnings += "\n" return warnings - - def _get_separator_list(self, separator, separator_2=' '): + def _get_separator_list(self, separator, separator_2=" "): """Creates column view for seperators in the TP/combined view.""" separator_list = [] for i in range(len(self._machine_model.get_ports()) - 1): - match_1 = re.search(r'\d+', self._machine_model.get_ports()[i]) - match_2 = re.search(r'\d+', self._machine_model.get_ports()[i + 1]) + match_1 = re.search(r"\d+", self._machine_model.get_ports()[i]) + match_2 = re.search(r"\d+", self._machine_model.get_ports()[i + 1]) if match_1 is not None and match_2 is not None and match_1.group() == match_2.group(): separator_list.append(separator_2) else: @@ -342,93 +348,93 @@ class Frontend(object): def _get_flag_symbols(self, flag_obj): """Returns flags for a flag object of an instruction""" - string_result = '' - string_result += '*' if INSTR_FLAGS.NOT_BOUND in flag_obj else '' - string_result += 'X' if INSTR_FLAGS.TP_UNKWN in flag_obj else '' - string_result += 'P' if INSTR_FLAGS.HIDDEN_LD in flag_obj else '' + string_result = "" + string_result += "*" if INSTR_FLAGS.NOT_BOUND in flag_obj else "" + string_result += "X" if INSTR_FLAGS.TP_UNKWN in flag_obj else "" + string_result += "P" if INSTR_FLAGS.HIDDEN_LD in flag_obj else "" # TODO add other flags - string_result += ' ' if len(string_result) == 0 else '' + string_result += " " if len(string_result) == 0 else "" return string_result - def _get_port_pressure(self, ports, port_len, used_ports=[], separator='|'): + def _get_port_pressure(self, ports, port_len, used_ports=[], separator="|"): """Returns line of port pressure for an instruction.""" if not isinstance(separator, list): separator = [separator for x in ports] - string_result = '{} '.format(separator[-1]) + string_result = "{} ".format(separator[-1]) for i in range(len(ports)): if float(ports[i]) == 0.0 and self._machine_model.get_ports()[i] not in used_ports: - string_result += port_len[i] * ' ' + ' {} '.format(separator[i]) + string_result += port_len[i] * " " + " {} ".format(separator[i]) continue - left_len = len(str(float(ports[i])).split('.')[0]) - substr = '{:' + str(left_len) + '.' + str(max(port_len[i] - left_len - 1, 0)) + 'f}' + left_len = len(str(float(ports[i])).split(".")[0]) + substr = "{:" + str(left_len) + "." + str(max(port_len[i] - left_len - 1, 0)) + "f}" substr = substr.format(ports[i]) string_result += ( - substr + ' {} '.format(separator[i]) - if '.' in substr - else '{:.1f}{} '.format(ports[i], separator[i]) + substr + " {} ".format(separator[i]) + if "." in substr + else "{:.1f}{} ".format(ports[i], separator[i]) ) return string_result[:-1] def _get_node_by_lineno(self, lineno, kernel): """Returns instruction form from kernel by its line number.""" - nodes = [instr for instr in kernel if instr['line_number'] == lineno] + nodes = [instr for instr in kernel if instr["line_number"] == lineno] return nodes[0] if len(nodes) > 0 else None - def _get_lcd_cp_ports(self, line_number, cp_dg, dependency, separator='|'): + def _get_lcd_cp_ports(self, line_number, cp_dg, dependency, separator="|"): """Returns the CP and LCD line for one instruction.""" - lat_cp = lat_lcd = '' + lat_cp = lat_lcd = "" if cp_dg: - lat_cp = float(self._get_node_by_lineno(line_number, cp_dg)['latency_cp']) + lat_cp = float(self._get_node_by_lineno(line_number, cp_dg)["latency_cp"]) if dependency: lat_lcd = float( - self._get_node_by_lineno(line_number, dependency['dependencies'])['latency_lcd'] + self._get_node_by_lineno(line_number, dependency["dependencies"])["latency_lcd"] ) - return '{} {:>4} {} {:>4} {}'.format(separator, lat_cp, separator, lat_lcd, separator) + return "{} {:>4} {} {:>4} {}".format(separator, lat_cp, separator, lat_lcd, separator) def _get_max_port_len(self, kernel): """Returns the maximal length needed to print all throughputs of the kernel.""" port_len = [4 for x in self._machine_model.get_ports()] for instruction_form in kernel: - for i, port in enumerate(instruction_form['port_pressure']): - if len('{:.2f}'.format(port)) > port_len[i]: - port_len[i] = len('{:.2f}'.format(port)) + for i, port in enumerate(instruction_form["port_pressure"]): + if len("{:.2f}".format(port)) > port_len[i]: + port_len[i] = len("{:.2f}".format(port)) return port_len - def _get_port_number_line(self, port_len, separator='|'): + def _get_port_number_line(self, port_len, separator="|"): """Returns column view of port identificators of machine_model.""" string_result = separator - separator_list = self._get_separator_list(separator, '-') + separator_list = self._get_separator_list(separator, "-") for i, length in enumerate(port_len): - substr = '{:^' + str(length + 2) + 's}' + substr = "{:^" + str(length + 2) + "s}" string_result += substr.format(self._machine_model.get_ports()[i]) + separator_list[i] return string_result def _header_report(self): """Prints header information""" - version = _get_version('__init__.py') + version = _get_version("__init__.py") adjust = 20 - header = '' - header += 'Open Source Architecture Code Analyzer (OSACA) - {}\n'.format(version) - header += 'Analyzed file:'.ljust(adjust) + '{}\n'.format(self._filename) - header += 'Architecture:'.ljust(adjust) + '{}\n'.format(self._arch.upper()) - header += 'Timestamp:'.ljust(adjust) + '{}\n'.format( - dt.utcnow().strftime('%Y-%m-%d %H:%M:%S') + header = "" + header += "Open Source Architecture Code Analyzer (OSACA) - {}\n".format(version) + header += "Analyzed file:".ljust(adjust) + "{}\n".format(self._filename) + header += "Architecture:".ljust(adjust) + "{}\n".format(self._arch.upper()) + header += "Timestamp:".ljust(adjust) + "{}\n".format( + dt.utcnow().strftime("%Y-%m-%d %H:%M:%S") ) - return header + '\n' + return header + "\n" def _symbol_map(self): """Prints instruction flag map.""" symbol_dict = { - INSTR_FLAGS.NOT_BOUND: 'Instruction micro-ops not bound to a port', - INSTR_FLAGS.TP_UNKWN: 'No throughput/latency information for this instruction in ' - + 'data file', - INSTR_FLAGS.HIDDEN_LD: 'Throughput of LOAD operation can be hidden behind a past ' - + 'or future STORE instruction', + INSTR_FLAGS.NOT_BOUND: "Instruction micro-ops not bound to a port", + INSTR_FLAGS.TP_UNKWN: "No throughput/latency information for this instruction in " + + "data file", + INSTR_FLAGS.HIDDEN_LD: "Throughput of LOAD operation can be hidden behind a past " + + "or future STORE instruction", } - symbol_map = '' + symbol_map = "" for flag in sorted(symbol_dict.keys()): - symbol_map += ' {} - {}\n'.format(self._get_flag_symbols([flag]), symbol_dict[flag]) + symbol_map += " {} - {}\n".format(self._get_flag_symbols([flag]), symbol_dict[flag]) return symbol_map - + def _port_binding_summary(self): raise NotImplementedError diff --git a/osaca/osaca.py b/osaca/osaca.py index 173b680..baa6aa7 100755 --- a/osaca/osaca.py +++ b/osaca/osaca.py @@ -10,27 +10,26 @@ import traceback from osaca.db_interface import import_benchmark_output, sanity_check from osaca.frontend import Frontend from osaca.parser import BaseParser, ParserAArch64, ParserX86ATT -from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG, - MachineModel, reduce_to_section) +from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section SUPPORTED_ARCHS = [ - 'SNB', - 'IVB', - 'HSW', - 'BDW', - 'SKX', - 'CSX', - 'ICL', - 'ZEN1', - 'ZEN2', - 'TX2', - 'N1', - 'A64FX', + "SNB", + "IVB", + "HSW", + "BDW", + "SKX", + "CSX", + "ICL", + "ZEN1", + "ZEN2", + "TX2", + "N1", + "A64FX", ] DEFAULT_ARCHS = { - 'aarch64': 'A64FX', - 'x86': 'SKX', + "aarch64": "A64FX", + "x86": "SKX", } @@ -50,7 +49,7 @@ def __find_version(*file_paths): version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) if version_match: return version_match.group(1) - raise RuntimeError('Unable to find version string.') + raise RuntimeError("Unable to find version string.") def get_version(): @@ -59,7 +58,7 @@ def get_version(): :returns: str -- the version string. """ - return __find_version('__init__.py') + return __find_version("__init__.py") def create_parser(parser=None): @@ -73,91 +72,92 @@ def create_parser(parser=None): # Create parser if not parser: parser = argparse.ArgumentParser( - description='Analyzes a marked innermost loop snippet for a given architecture type.', - epilog='For help, examples, documentation and bug reports go to:\nhttps://github.com' - '/RRZE-HPC/OSACA/ | License: AGPLv3', + description="Analyzes a marked innermost loop snippet for a given architecture type.", + epilog="For help, examples, documentation and bug reports go to:\nhttps://github.com" + "/RRZE-HPC/OSACA/ | License: AGPLv3", ) # Add arguments parser.add_argument( - '-V', '--version', action='version', version='%(prog)s ' + __find_version('__init__.py') + "-V", "--version", action="version", version="%(prog)s " + __find_version("__init__.py") ) parser.add_argument( - '--arch', + "--arch", type=str, - help='Define architecture (SNB, IVB, HSW, BDW, SKX, CSX, ICL, ZEN1, ZEN2, TX2, N1, ' - 'A64FX). If no architecture is given, OSACA assumes a default uarch for x86/AArch64.', + help="Define architecture (SNB, IVB, HSW, BDW, SKX, CSX, ICL, ZEN1, ZEN2, TX2, N1, " + "A64FX). If no architecture is given, OSACA assumes a default uarch for x86/AArch64.", ) parser.add_argument( - '--fixed', - action='store_true', - help='Run the throughput analysis with fixed probabilities for all suitable ports per ' - 'instruction. Otherwise, OSACA will print the optimal port utilization for the kernel.', + "--fixed", + action="store_true", + help="Run the throughput analysis with fixed probabilities for all suitable ports per " + "instruction. Otherwise, OSACA will print the optimal port utilization for the kernel.", ) parser.add_argument( - '--lines', + "--lines", type=str, - help='Define lines that should be included in the analysis. This option overwrites any' - ' range defined by markers in the assembly. Add either single lines or ranges defined by' + help="Define lines that should be included in the analysis. This option overwrites any" + " range defined by markers in the assembly. Add either single lines or ranges defined by" ' "-" or ":", each entry separated by commas, e.g.: --lines 1,2,8-18,20:24', ) parser.add_argument( - '--db-check', - dest='check_db', - action='store_true', + "--db-check", + dest="check_db", + action="store_true", help='Run a sanity check on the by "--arch" specified database. The output depends ' - 'on the verbosity level.', + "on the verbosity level.", ) parser.add_argument( - '--online', - dest='internet_check', - action='store_true', - help='Run sanity check with online DB validation (currently felixcloutier) to see the ' - 'src/dst distribution of the operands. Can be only used in combination with --db-check.', + "--online", + dest="internet_check", + action="store_true", + help="Run sanity check with online DB validation (currently felixcloutier) to see the " + "src/dst distribution of the operands. Can be only used in combination with --db-check.", ) parser.add_argument( - '--import', - metavar='MICROBENCH', - dest='import_data', + "--import", + metavar="MICROBENCH", + dest="import_data", type=str, default=argparse.SUPPRESS, - help='Import a given microbenchmark output file into the corresponding architecture ' + help="Import a given microbenchmark output file into the corresponding architecture " 'instruction database. Define the type of microbenchmark either as "ibench" or ' '"asmbench".', ) parser.add_argument( - '--insert-marker', - dest='insert_marker', - action='store_true', - help='Try to find assembly block containing the loop to analyse and insert byte ' - 'marker by using Kerncraft.', + "--insert-marker", + dest="insert_marker", + action="store_true", + help="Try to find assembly block containing the loop to analyse and insert byte " + "marker by using Kerncraft.", ) parser.add_argument( - '--export-graph', - metavar='EXPORT_PATH', - dest='dotpath', + "--export-graph", + metavar="EXPORT_PATH", + dest="dotpath", default=None, type=str, help='Output path for .dot file export. If "." is given, the file will be stored as ' '"./osaca_dg.dot"', ) parser.add_argument( - '--ignore-unknown', - dest='ignore_unknown', - action='store_true', - help='Ignore if instructions cannot be found in the data file and print analysis anyway.', + "--ignore-unknown", + dest="ignore_unknown", + action="store_true", + help="Ignore if instructions cannot be found in the data file and print analysis anyway.", ) parser.add_argument( - '--verbose', '-v', action='count', default=0, help='Increases verbosity level.' + "--verbose", "-v", action="count", default=0, help="Increases verbosity level." ) parser.add_argument( - '--out', '-o', + "--out", + "-o", default=sys.stdout, - type=argparse.FileType('w'), - help='Write analysis to this file (default to stdout).' + type=argparse.FileType("w"), + help="Write analysis to this file (default to stdout).", ) parser.add_argument( - 'file', type=argparse.FileType('r'), help='Path to object (ASM or instruction file).' + "file", type=argparse.FileType("r"), help="Path to object (ASM or instruction file)." ) return parser @@ -170,24 +170,24 @@ def check_arguments(args, parser): :param args: arguments given from :class:`~argparse.ArgumentParser` after parsing :param parser: :class:`~argparse.ArgumentParser` object """ - supported_import_files = ['ibench', 'asmbench'] + supported_import_files = ["ibench", "asmbench"] - if args.arch is None and (args.check_db or 'import_data' in args): + if args.arch is None and (args.check_db or "import_data" in args): parser.error( - 'DB check and data import cannot work with a default microarchitecture. ' - 'Please see --help for all valid architecture codes.' + "DB check and data import cannot work with a default microarchitecture. " + "Please see --help for all valid architecture codes." ) elif args.arch is not None and args.arch.upper() not in SUPPORTED_ARCHS: parser.error( - 'Microarchitecture not supported. Please see --help for all valid architecture codes.' + "Microarchitecture not supported. Please see --help for all valid architecture codes." ) - if 'import_data' in args and args.import_data not in supported_import_files: + if "import_data" in args and args.import_data not in supported_import_files: parser.error( - 'Microbenchmark not supported for data import. Please see --help for all valid ' - 'microbenchmark codes.' + "Microbenchmark not supported for data import. Please see --help for all valid " + "microbenchmark codes." ) if args.internet_check and not args.check_db: - parser.error('--online requires --check-db') + parser.error("--online requires --check-db") def import_data(benchmark_type, arch, filepath, output_file=sys.stdout): @@ -203,12 +203,12 @@ def import_data(benchmark_type, arch, filepath, output_file=sys.stdout): :param output_file: output stream specifying where to write output, defaults to :class:`sys.stdout` :type output_file: stream, optional """ - if benchmark_type.lower() == 'ibench': - import_benchmark_output(arch, 'ibench', filepath, output=output_file) - elif benchmark_type.lower() == 'asmbench': - import_benchmark_output(arch, 'asmbench', filepath, output=output_file) + if benchmark_type.lower() == "ibench": + import_benchmark_output(arch, "ibench", filepath, output=output_file) + elif benchmark_type.lower() == "asmbench": + import_benchmark_output(arch, "asmbench", filepath, output=output_file) else: - raise NotImplementedError('This benchmark input variant is not supported.') + raise NotImplementedError("This benchmark input variant is not supported.") def insert_byte_marker(args): @@ -221,9 +221,9 @@ def insert_byte_marker(args): from kerncraft.incore_model import asm_instrumentation except ImportError: print( - 'Module kerncraft not installed. Use \'pip install --user ' - 'kerncraft\' for installation.\nFor more information see ' - 'https://github.com/RRZE-HPC/kerncraft', + "Module kerncraft not installed. Use 'pip install --user " + "kerncraft' for installation.\nFor more information see " + "https://github.com/RRZE-HPC/kerncraft", file=sys.stderr, ) sys.exit(1) @@ -234,14 +234,14 @@ def insert_byte_marker(args): asm_instrumentation( input_file=unmarked_assembly, output_file=marked_assembly, - block_selection='manual', - pointer_increment='auto_with_manual_fallback', + block_selection="manual", + pointer_increment="auto_with_manual_fallback", isa=MachineModel.get_isa_for_arch(args.arch), ) marked_assembly.seek(0) assembly = marked_assembly.read() - with open(args.file.name, 'w') as f: + with open(args.file.name, "w") as f: f.write(assembly) @@ -272,7 +272,11 @@ def inspect(args, output_file=sys.stdout): # probably the wrong parser based on heuristic if args.arch is None: # change ISA and try again - arch = DEFAULT_ARCHS['x86'] if BaseParser.detect_ISA(code) == 'aarch64' else DEFAULT_ARCHS['aarch64'] + arch = ( + DEFAULT_ARCHS["x86"] + if BaseParser.detect_ISA(code) == "aarch64" + else DEFAULT_ARCHS["aarch64"] + ) isa = MachineModel.get_isa_for_arch(arch) parser = get_asm_parser(arch) parsed_code = parser.parse_file(code) @@ -282,12 +286,14 @@ def inspect(args, output_file=sys.stdout): # Reduce to marked kernel or chosen section and add semantics if args.lines: line_range = get_line_range(args.lines) - kernel = [line for line in parsed_code if line['line_number'] in line_range] + kernel = [line for line in parsed_code if line["line_number"] in line_range] print_length_warning = False else: kernel = reduce_to_section(parsed_code, isa) # Print warning if kernel has no markers and is larger than threshold (100) - print_length_warning = True if len(kernel) == len(parsed_code) and len(kernel) > 100 else False + print_length_warning = ( + True if len(kernel) == len(parsed_code) and len(kernel) > 100 else False + ) machine_model = MachineModel(arch=arch) semantics = ArchSemantics(machine_model) semantics.add_semantics(kernel) @@ -298,7 +304,7 @@ def inspect(args, output_file=sys.stdout): # Create DiGrahps kernel_graph = KernelDG(kernel, parser, machine_model) if args.dotpath is not None: - kernel_graph.export_graph(args.dotpath if args.dotpath != '.' else None) + kernel_graph.export_graph(args.dotpath if args.dotpath != "." else None) # Print analysis frontend = Frontend(args.file.name, arch=arch) print( @@ -308,7 +314,7 @@ def inspect(args, output_file=sys.stdout): ignore_unknown=ignore_unknown, arch_warning=print_arch_warning, length_warning=print_length_warning, - verbose=verbose + verbose=verbose, ), file=output_file, ) @@ -328,7 +334,7 @@ def run(args, output_file=sys.stdout): sanity_check( args.arch, verbose=verbose, internet_check=args.internet_check, output_file=output_file ) - elif 'import_data' in args: + elif "import_data" in args: # Import microbench output file into DB import_data(args.import_data, args.arch, args.file.name, output_file=output_file) elif args.insert_marker: @@ -348,9 +354,9 @@ def get_asm_parser(arch) -> BaseParser: :returns: :class:`~osaca.parser.BaseParser` object """ isa = MachineModel.get_isa_for_arch(arch) - if isa == 'x86': + if isa == "x86": return ParserX86ATT() - elif isa == 'aarch64': + elif isa == "aarch64": return ParserAArch64() @@ -359,26 +365,28 @@ def get_unmatched_instruction_ratio(kernel): unmatched_counter = 0 for instruction in kernel: if ( - INSTR_FLAGS.TP_UNKWN in instruction['flags'] - and INSTR_FLAGS.LT_UNKWN in instruction['flags'] + INSTR_FLAGS.TP_UNKWN in instruction["flags"] + and INSTR_FLAGS.LT_UNKWN in instruction["flags"] ): unmatched_counter += 1 return unmatched_counter / len(kernel) + def get_line_range(line_str): - line_str = line_str.replace(':', '-') - lines = line_str.split(',') + line_str = line_str.replace(":", "-") + lines = line_str.split(",") lines_int = [] for l in lines: - if '-' in l: - start = int(l.split('-')[0]) - end = int(l.split('-')[1]) - rnge = list(range(start, end+1)) + if "-" in l: + start = int(l.split("-")[0]) + end = int(l.split("-")[1]) + rnge = list(range(start, end + 1)) lines_int += rnge else: lines_int.append(int(l)) return lines_int + def main(): """Initialize and run command line interface.""" parser = create_parser() @@ -387,5 +395,5 @@ def main(): run(args, output_file=args.out) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/osaca/parser/__init__.py b/osaca/parser/__init__.py index 1e9d6c6..ba43c70 100644 --- a/osaca/parser/__init__.py +++ b/osaca/parser/__init__.py @@ -8,12 +8,13 @@ from .base_parser import BaseParser from .parser_x86att import ParserX86ATT from .parser_AArch64 import ParserAArch64 -__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64', 'get_parser'] +__all__ = ["AttrDict", "BaseParser", "ParserX86ATT", "ParserAArch64", "get_parser"] + def get_parser(isa): - if isa.lower() == 'x86': + if isa.lower() == "x86": return ParserX86ATT() - elif isa.lower() == 'aarch64': + elif isa.lower() == "aarch64": return ParserAArch64() else: raise ValueError("Unknown ISA {!r}.".format(isa)) diff --git a/osaca/parser/base_parser.py b/osaca/parser/base_parser.py index ced8fa6..b55c9b7 100755 --- a/osaca/parser/base_parser.py +++ b/osaca/parser/base_parser.py @@ -3,18 +3,19 @@ import operator import re + class BaseParser(object): # Identifiers for operand types - COMMENT_ID = 'comment' - DIRECTIVE_ID = 'directive' - IMMEDIATE_ID = 'immediate' - LABEL_ID = 'label' - IDENTIFIER_ID = 'identifier' - MEMORY_ID = 'memory' - REGISTER_ID = 'register' - SEGMENT_EXT_ID = 'segment_extension' - INSTRUCTION_ID = 'instruction' - OPERANDS_ID = 'operands' + COMMENT_ID = "comment" + DIRECTIVE_ID = "directive" + IMMEDIATE_ID = "immediate" + LABEL_ID = "label" + IDENTIFIER_ID = "identifier" + MEMORY_ID = "memory" + REGISTER_ID = "register" + SEGMENT_EXT_ID = "segment_extension" + INSTRUCTION_ID = "instruction" + OPERANDS_ID = "operands" _parser_constructed = False def __init__(self): @@ -27,15 +28,15 @@ class BaseParser(object): """Detect the ISA of the assembly based on the used registers and return the ISA code.""" # Check for the amount of registers in the code to determine the ISA # 1) Check for xmm, ymm, zmm, rax, rbx, rcx, and rdx registers in x86 - heuristics_x86ATT = [r'%[xyz]mm[0-9]', r'%[er][abcd]x[0-9]'] + heuristics_x86ATT = [r"%[xyz]mm[0-9]", r"%[er][abcd]x[0-9]"] # 2) check for v and z vector registers and x/w general-purpose registers - heuristics_aarch64 = [r'[vz][0-9][0-9]?\.[0-9][0-9]?[bhsd]', r'[wx][0-9]'] - matches = {'x86': 0, 'aarch64': 0} + heuristics_aarch64 = [r"[vz][0-9][0-9]?\.[0-9][0-9]?[bhsd]", r"[wx][0-9]"] + matches = {"x86": 0, "aarch64": 0} for h in heuristics_x86ATT: - matches['x86'] += len(re.findall(h, file_content)) + matches["x86"] += len(re.findall(h, file_content)) for h in heuristics_aarch64: - matches['aarch64'] += len(re.findall(h, file_content)) + matches["aarch64"] += len(re.findall(h, file_content)) return max(matches.items(), key=operator.itemgetter(1))[0] @@ -50,9 +51,9 @@ class BaseParser(object): """ # Create instruction form list asm_instructions = [] - lines = file_content.split('\n') + lines = file_content.split("\n") for i, line in enumerate(lines): - if line.strip() == '': + if line.strip() == "": continue asm_instructions.append(self.parse_line(line, i + 1 + start_line)) return asm_instructions diff --git a/osaca/parser/parser_AArch64.py b/osaca/parser/parser_AArch64.py index d1a483b..ceae81b 100755 --- a/osaca/parser/parser_AArch64.py +++ b/osaca/parser/parser_AArch64.py @@ -17,53 +17,56 @@ class ParserAArch64(BaseParser): def __init__(self): super().__init__() - self.isa = 'aarch64' + self.isa = "aarch64" def construct_parser(self): """Create parser for ARM AArch64 ISA.""" # Comment - symbol_comment = '//' + symbol_comment = "//" self.comment = pp.Literal(symbol_comment) + pp.Group( pp.ZeroOrMore(pp.Word(pp.printables)) ).setResultsName(self.COMMENT_ID) # Define ARM assembly identifier - decimal_number = pp.Combine( - pp.Optional(pp.Literal('-')) + pp.Word(pp.nums) - ).setResultsName('value') - hex_number = pp.Combine(pp.Literal('0x') + pp.Word(pp.hexnums)).setResultsName('value') - relocation = pp.Combine(pp.Literal(':') + pp.Word(pp.alphanums + '_') + pp.Literal(':')) - first = pp.Word(pp.alphas + '_.', exact=1) - rest = pp.Word(pp.alphanums + '_.') + decimal_number = pp.Combine(pp.Optional(pp.Literal("-")) + pp.Word(pp.nums)).setResultsName( + "value" + ) + hex_number = pp.Combine(pp.Literal("0x") + pp.Word(pp.hexnums)).setResultsName("value") + relocation = pp.Combine(pp.Literal(":") + pp.Word(pp.alphanums + "_") + pp.Literal(":")) + first = pp.Word(pp.alphas + "_.", exact=1) + rest = pp.Word(pp.alphanums + "_.") identifier = pp.Group( - pp.Optional(relocation).setResultsName('relocation') - + pp.Combine(first + pp.Optional(rest)).setResultsName('name') - + pp.Optional(pp.Suppress(pp.Literal('+')) + (hex_number | decimal_number).setResultsName('offset')) + pp.Optional(relocation).setResultsName("relocation") + + pp.Combine(first + pp.Optional(rest)).setResultsName("name") + + pp.Optional( + pp.Suppress(pp.Literal("+")) + + (hex_number | decimal_number).setResultsName("offset") + ) ).setResultsName(self.IDENTIFIER_ID) # Label self.label = pp.Group( - identifier.setResultsName('name') + pp.Literal(':') + pp.Optional(self.comment) + identifier.setResultsName("name") + pp.Literal(":") + pp.Optional(self.comment) ).setResultsName(self.LABEL_ID) # Directive directive_option = pp.Combine( - pp.Word(pp.alphas + '#@.%', exact=1) - + pp.Optional(pp.Word(pp.printables + ' ', excludeChars=',')) + pp.Word(pp.alphas + "#@.%", exact=1) + + pp.Optional(pp.Word(pp.printables + " ", excludeChars=",")) ) directive_parameter = ( pp.quotedString | directive_option | identifier | hex_number | decimal_number ) - commaSeparatedList = pp.delimitedList(pp.Optional(directive_parameter), delim=',') + commaSeparatedList = pp.delimitedList(pp.Optional(directive_parameter), delim=",") self.directive = pp.Group( - pp.Literal('.') - + pp.Word(pp.alphanums + '_').setResultsName('name') - + (pp.OneOrMore(directive_parameter) ^ commaSeparatedList).setResultsName('parameters') + pp.Literal(".") + + pp.Word(pp.alphanums + "_").setResultsName("name") + + (pp.OneOrMore(directive_parameter) ^ commaSeparatedList).setResultsName("parameters") + pp.Optional(self.comment) ).setResultsName(self.DIRECTIVE_ID) # LLVM-MCA markers self.llvm_markers = pp.Group( - pp.Literal('#') + pp.Literal("#") + pp.Combine( - pp.CaselessLiteral('LLVM-MCA-') - + (pp.CaselessLiteral('BEGIN') | pp.CaselessLiteral('END')) + pp.CaselessLiteral("LLVM-MCA-") + + (pp.CaselessLiteral("BEGIN") | pp.CaselessLiteral("END")) ) + pp.Optional(self.comment) ).setResultsName(self.COMMENT_ID) @@ -72,41 +75,41 @@ class ParserAArch64(BaseParser): # Instructions # Mnemonic # (?P[a-zA-Z][a-zA-Z0-9]*)(?PS?)(P?.[a-zA-Z]{2}) - mnemonic = pp.Word(pp.alphanums + '.').setResultsName('mnemonic') + mnemonic = pp.Word(pp.alphanums + ".").setResultsName("mnemonic") # Immediate: # int: ^-?[0-9]+ | hex: ^0x[0-9a-fA-F]+ | fp: ^[0-9]{1}.[0-9]+[eE]{1}[\+-]{1}[0-9]+[fF]? - symbol_immediate = '#' + symbol_immediate = "#" mantissa = pp.Combine( - pp.Optional(pp.Literal('-')) + pp.Word(pp.nums) + pp.Literal('.') + pp.Word(pp.nums) - ).setResultsName('mantissa') + pp.Optional(pp.Literal("-")) + pp.Word(pp.nums) + pp.Literal(".") + pp.Word(pp.nums) + ).setResultsName("mantissa") exponent = ( - pp.CaselessLiteral('e') - + pp.Word('+-').setResultsName('e_sign') - + pp.Word(pp.nums).setResultsName('exponent') + pp.CaselessLiteral("e") + + pp.Word("+-").setResultsName("e_sign") + + pp.Word(pp.nums).setResultsName("exponent") ) float_ = pp.Group( - mantissa + pp.Optional(exponent) + pp.CaselessLiteral('f') - ).setResultsName('float') - double_ = pp.Group(mantissa + pp.Optional(exponent)).setResultsName('double') + mantissa + pp.Optional(exponent) + pp.CaselessLiteral("f") + ).setResultsName("float") + double_ = pp.Group(mantissa + pp.Optional(exponent)).setResultsName("double") immediate = pp.Group( pp.Optional(pp.Literal(symbol_immediate)) + (hex_number ^ decimal_number ^ float_ ^ double_) | (pp.Optional(pp.Literal(symbol_immediate)) + identifier) ).setResultsName(self.IMMEDIATE_ID) shift_op = ( - pp.CaselessLiteral('lsl') - ^ pp.CaselessLiteral('lsr') - ^ pp.CaselessLiteral('asr') - ^ pp.CaselessLiteral('ror') - ^ pp.CaselessLiteral('sxtw') - ^ pp.CaselessLiteral('uxtw') - ^ pp.CaselessLiteral('mul vl') + pp.CaselessLiteral("lsl") + ^ pp.CaselessLiteral("lsr") + ^ pp.CaselessLiteral("asr") + ^ pp.CaselessLiteral("ror") + ^ pp.CaselessLiteral("sxtw") + ^ pp.CaselessLiteral("uxtw") + ^ pp.CaselessLiteral("mul vl") ) arith_immediate = pp.Group( - immediate.setResultsName('base_immediate') - + pp.Suppress(pp.Literal(',')) - + shift_op.setResultsName('shift_op') - + pp.Optional(immediate).setResultsName('shift') + immediate.setResultsName("base_immediate") + + pp.Suppress(pp.Literal(",")) + + shift_op.setResultsName("shift_op") + + pp.Optional(immediate).setResultsName("shift") ).setResultsName(self.IMMEDIATE_ID) # Register: # scalar: [XWBHSDQ][0-9]{1,2} | vector: [VZ][0-9]{1,2}(\.[12468]{1,2}[BHSD])? @@ -114,83 +117,81 @@ class ParserAArch64(BaseParser): # ignore vector len control ZCR_EL[123] for now # define SP, ZR register aliases as regex, due to pyparsing does not support # proper lookahead - alias_r31_sp = pp.Regex('(?P[a-zA-Z])?(?P(sp|SP))') - alias_r31_zr = pp.Regex('(?P[a-zA-Z])?(?P(zr|ZR))') - scalar = pp.Word('xwbhsdqXWBHSDQ', exact=1).setResultsName('prefix') + pp.Word( + alias_r31_sp = pp.Regex("(?P[a-zA-Z])?(?P(sp|SP))") + alias_r31_zr = pp.Regex("(?P[a-zA-Z])?(?P(zr|ZR))") + scalar = pp.Word("xwbhsdqXWBHSDQ", exact=1).setResultsName("prefix") + pp.Word( pp.nums - ).setResultsName('name') - index = pp.Literal('[') + pp.Word(pp.nums).setResultsName('index') + pp.Literal(']') + ).setResultsName("name") + index = pp.Literal("[") + pp.Word(pp.nums).setResultsName("index") + pp.Literal("]") vector = ( - pp.oneOf('v z', caseless=True).setResultsName('prefix') - + pp.Word(pp.nums).setResultsName('name') + pp.oneOf("v z", caseless=True).setResultsName("prefix") + + pp.Word(pp.nums).setResultsName("name") + pp.Optional( - pp.Literal('.') - + pp.Optional(pp.Word('12468')).setResultsName('lanes') - + pp.Word(pp.alphas, exact=1).setResultsName('shape') + pp.Literal(".") + + pp.Optional(pp.Word("12468")).setResultsName("lanes") + + pp.Word(pp.alphas, exact=1).setResultsName("shape") + pp.Optional(index) ) ) predicate = ( - pp.CaselessLiteral('p').setResultsName('prefix') - + pp.Word(pp.nums).setResultsName('name') + pp.CaselessLiteral("p").setResultsName("prefix") + + pp.Word(pp.nums).setResultsName("name") + pp.Optional( ( - pp.Suppress(pp.Literal('/')) - + pp.oneOf('z m', caseless=True).setResultsName('predication') + pp.Suppress(pp.Literal("/")) + + pp.oneOf("z m", caseless=True).setResultsName("predication") ) | ( - pp.Literal('.') - + pp.Optional(pp.Word('12468')).setResultsName('lanes') - + pp.Word(pp.alphas, exact=1).setResultsName('shape') + pp.Literal(".") + + pp.Optional(pp.Word("12468")).setResultsName("lanes") + + pp.Word(pp.alphas, exact=1).setResultsName("shape") ) ) ) self.list_element = vector ^ scalar register_list = ( - pp.Literal('{') + pp.Literal("{") + ( - pp.delimitedList(pp.Combine(self.list_element), delim=',').setResultsName('list') - ^ pp.delimitedList(pp.Combine(self.list_element), delim='-').setResultsName( - 'range' - ) + pp.delimitedList(pp.Combine(self.list_element), delim=",").setResultsName("list") + ^ pp.delimitedList(pp.Combine(self.list_element), delim="-").setResultsName("range") ) - + pp.Literal('}') + + pp.Literal("}") + pp.Optional(index) ) register = pp.Group( (alias_r31_sp | alias_r31_zr | vector | scalar | predicate | register_list) - #(alias_r31_sp | alias_r31_zr | vector | scalar | predicate | register_list) + # (alias_r31_sp | alias_r31_zr | vector | scalar | predicate | register_list) + pp.Optional( - pp.Suppress(pp.Literal(',')) - + shift_op.setResultsName('shift_op') - + pp.Optional(immediate).setResultsName('shift') + pp.Suppress(pp.Literal(",")) + + shift_op.setResultsName("shift_op") + + pp.Optional(immediate).setResultsName("shift") ) ).setResultsName(self.REGISTER_ID) self.register = register # Memory - register_index = register.setResultsName('index') + pp.Optional( - pp.Literal(',') + pp.Word(pp.alphas) + immediate.setResultsName('scale') + register_index = register.setResultsName("index") + pp.Optional( + pp.Literal(",") + pp.Word(pp.alphas) + immediate.setResultsName("scale") ) memory = pp.Group( - pp.Literal('[') - + pp.Optional(register.setResultsName('base')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(register_index ^ (immediate ^ arith_immediate).setResultsName('offset')) - + pp.Literal(']') + pp.Literal("[") + + pp.Optional(register.setResultsName("base")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(register_index ^ (immediate ^ arith_immediate).setResultsName("offset")) + + pp.Literal("]") + pp.Optional( - pp.Literal('!').setResultsName('pre_indexed') - | (pp.Suppress(pp.Literal(',')) + immediate.setResultsName('post_indexed')) + pp.Literal("!").setResultsName("pre_indexed") + | (pp.Suppress(pp.Literal(",")) + immediate.setResultsName("post_indexed")) ) ).setResultsName(self.MEMORY_ID) prefetch_op = pp.Group( - pp.Group(pp.CaselessLiteral('PLD') ^ pp.CaselessLiteral('PST')).setResultsName('type') + pp.Group(pp.CaselessLiteral("PLD") ^ pp.CaselessLiteral("PST")).setResultsName("type") + pp.Group( - pp.CaselessLiteral('L1') ^ pp.CaselessLiteral('L2') ^ pp.CaselessLiteral('L3') - ).setResultsName('target') - + pp.Group(pp.CaselessLiteral('KEEP') ^ pp.CaselessLiteral('STRM')).setResultsName( - 'policy' + pp.CaselessLiteral("L1") ^ pp.CaselessLiteral("L2") ^ pp.CaselessLiteral("L3") + ).setResultsName("target") + + pp.Group(pp.CaselessLiteral("KEEP") ^ pp.CaselessLiteral("STRM")).setResultsName( + "policy" ) - ).setResultsName('prfop') + ).setResultsName("prfop") # Combine to instruction form operand_first = pp.Group( register ^ (prefetch_op | immediate) ^ memory ^ arith_immediate ^ identifier @@ -198,15 +199,15 @@ class ParserAArch64(BaseParser): operand_rest = pp.Group((register ^ immediate ^ memory ^ arith_immediate) | identifier) self.instruction_parser = ( mnemonic - + pp.Optional(operand_first.setResultsName('operand1')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand2')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand3')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand4')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand5')) + + pp.Optional(operand_first.setResultsName("operand1")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand2")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand3")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand4")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand5")) + pp.Optional(self.comment) ) @@ -231,8 +232,8 @@ class ParserAArch64(BaseParser): self.DIRECTIVE_ID: None, self.COMMENT_ID: None, self.LABEL_ID: None, - 'line': line, - 'line_number': line_number, + "line": line, + "line_number": line_number, } ) result = None @@ -241,7 +242,7 @@ class ParserAArch64(BaseParser): try: result = self.process_operand(self.comment.parseString(line, parseAll=True).asDict()) result = AttrDict.convert_dict(result) - instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID]) + instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID]) except pp.ParseException: pass # 1.2 check for llvm-mca marker @@ -250,7 +251,7 @@ class ParserAArch64(BaseParser): self.llvm_markers.parseString(line, parseAll=True).asDict() ) result = AttrDict.convert_dict(result) - instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID]) + instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID]) except pp.ParseException: pass # 2. Parse label @@ -260,7 +261,7 @@ class ParserAArch64(BaseParser): result = AttrDict.convert_dict(result) instruction_form[self.LABEL_ID] = result[self.LABEL_ID].name if self.COMMENT_ID in result[self.LABEL_ID]: - instruction_form[self.COMMENT_ID] = ' '.join( + instruction_form[self.COMMENT_ID] = " ".join( result[self.LABEL_ID][self.COMMENT_ID] ) except pp.ParseException: @@ -275,12 +276,12 @@ class ParserAArch64(BaseParser): result = AttrDict.convert_dict(result) instruction_form[self.DIRECTIVE_ID] = AttrDict( { - 'name': result[self.DIRECTIVE_ID].name, - 'parameters': result[self.DIRECTIVE_ID].parameters, + "name": result[self.DIRECTIVE_ID].name, + "parameters": result[self.DIRECTIVE_ID].parameters, } ) if self.COMMENT_ID in result[self.DIRECTIVE_ID]: - instruction_form[self.COMMENT_ID] = ' '.join( + instruction_form[self.COMMENT_ID] = " ".join( result[self.DIRECTIVE_ID][self.COMMENT_ID] ) except pp.ParseException: @@ -292,8 +293,7 @@ class ParserAArch64(BaseParser): result = self.parse_instruction(line) except (pp.ParseException, KeyError) as e: raise e - raise ValueError( - 'Unable to parse {!r} on line {}'.format(line, line_number)) from e + raise ValueError("Unable to parse {!r} on line {}".format(line, line_number)) from e instruction_form[self.INSTRUCTION_ID] = result[self.INSTRUCTION_ID] instruction_form[self.OPERANDS_ID] = result[self.OPERANDS_ID] instruction_form[self.COMMENT_ID] = result[self.COMMENT_ID] @@ -312,26 +312,26 @@ class ParserAArch64(BaseParser): operands = [] # Add operands to list # Check first operand - if 'operand1' in result: - operands.append(self.process_operand(result['operand1'])) + if "operand1" in result: + operands.append(self.process_operand(result["operand1"])) # Check second operand - if 'operand2' in result: - operands.append(self.process_operand(result['operand2'])) + if "operand2" in result: + operands.append(self.process_operand(result["operand2"])) # Check third operand - if 'operand3' in result: - operands.append(self.process_operand(result['operand3'])) + if "operand3" in result: + operands.append(self.process_operand(result["operand3"])) # Check fourth operand - if 'operand4' in result: - operands.append(self.process_operand(result['operand4'])) + if "operand4" in result: + operands.append(self.process_operand(result["operand4"])) # Check fifth operand - if 'operand5' in result: - operands.append(self.process_operand(result['operand5'])) + if "operand5" in result: + operands.append(self.process_operand(result["operand5"])) return_dict = AttrDict( { self.INSTRUCTION_ID: result.mnemonic, self.OPERANDS_ID: operands, - self.COMMENT_ID: ' '.join(result[self.COMMENT_ID]) + self.COMMENT_ID: " ".join(result[self.COMMENT_ID]) if self.COMMENT_ID in result else None, } @@ -345,11 +345,11 @@ class ParserAArch64(BaseParser): return self.process_memory_address(operand[self.MEMORY_ID]) # structure register lists if self.REGISTER_ID in operand and ( - 'list' in operand[self.REGISTER_ID] or 'range' in operand[self.REGISTER_ID] + "list" in operand[self.REGISTER_ID] or "range" in operand[self.REGISTER_ID] ): # TODO: discuss if ranges should be converted to lists return self.process_register_list(operand[self.REGISTER_ID]) - if self.REGISTER_ID in operand and operand[self.REGISTER_ID]['name'] == 'sp': + if self.REGISTER_ID in operand and operand[self.REGISTER_ID]["name"] == "sp": return self.process_sp_register(operand[self.REGISTER_ID]) # add value attribute to floating point immediates without exponent if self.IMMEDIATE_ID in operand: @@ -363,140 +363,140 @@ class ParserAArch64(BaseParser): def process_memory_address(self, memory_address): """Post-process memory address operand""" # Remove unnecessarily created dictionary entries during parsing - offset = memory_address.get('offset', None) + offset = memory_address.get("offset", None) if isinstance(offset, list) and len(offset) == 1: offset = offset[0] - base = memory_address.get('base', None) - index = memory_address.get('index', None) + base = memory_address.get("base", None) + index = memory_address.get("index", None) scale = 1 - if base is not None and 'name' in base and base['name'] == 'sp': - base['prefix'] = 'x' - if index is not None and 'name' in index and index['name'] == 'sp': - index['prefix'] = 'x' - valid_shift_ops = ['lsl', 'uxtw', 'sxtw'] - if 'index' in memory_address: - if 'shift' in memory_address['index']: - if memory_address['index']['shift_op'].lower() in valid_shift_ops: - scale = 2 ** int(memory_address['index']['shift'][0]['value']) - new_dict = AttrDict({'offset': offset, 'base': base, 'index': index, 'scale': scale}) - if 'pre_indexed' in memory_address: - new_dict['pre_indexed'] = True - if 'post_indexed' in memory_address: - new_dict['post_indexed'] = memory_address['post_indexed'] + if base is not None and "name" in base and base["name"] == "sp": + base["prefix"] = "x" + if index is not None and "name" in index and index["name"] == "sp": + index["prefix"] = "x" + valid_shift_ops = ["lsl", "uxtw", "sxtw"] + if "index" in memory_address: + if "shift" in memory_address["index"]: + if memory_address["index"]["shift_op"].lower() in valid_shift_ops: + scale = 2 ** int(memory_address["index"]["shift"][0]["value"]) + new_dict = AttrDict({"offset": offset, "base": base, "index": index, "scale": scale}) + if "pre_indexed" in memory_address: + new_dict["pre_indexed"] = True + if "post_indexed" in memory_address: + new_dict["post_indexed"] = memory_address["post_indexed"] return AttrDict({self.MEMORY_ID: new_dict}) def process_sp_register(self, register): """Post-process stack pointer register""" reg = register - reg['prefix'] = 'x' + reg["prefix"] = "x" return AttrDict({self.REGISTER_ID: reg}) def process_register_list(self, register_list): """Post-process register lists (e.g., {r0,r3,r5}) and register ranges (e.g., {r0-r7})""" # Remove unnecessarily created dictionary entries during parsing rlist = [] - dict_name = '' - if 'list' in register_list: - dict_name = 'list' - if 'range' in register_list: - dict_name = 'range' + dict_name = "" + if "list" in register_list: + dict_name = "list" + if "range" in register_list: + dict_name = "range" for r in register_list[dict_name]: rlist.append( AttrDict.convert_dict(self.list_element.parseString(r, parseAll=True).asDict()) ) - index = register_list.get('index', None) - new_dict = AttrDict({dict_name: rlist, 'index': index}) + index = register_list.get("index", None) + new_dict = AttrDict({dict_name: rlist, "index": index}) if len(new_dict[dict_name]) == 1: return AttrDict({self.REGISTER_ID: new_dict[dict_name][0]}) return AttrDict({self.REGISTER_ID: new_dict}) def process_immediate(self, immediate): """Post-process immediate operand""" - dict_name = '' - if 'identifier' in immediate: + dict_name = "" + if "identifier" in immediate: # actually an identifier, change declaration return immediate - if 'value' in immediate: + if "value" in immediate: # normal integer value, nothing to do return AttrDict({self.IMMEDIATE_ID: immediate}) - if 'base_immediate' in immediate: + if "base_immediate" in immediate: # arithmetic immediate, add calculated value as value - immediate['shift'] = immediate['shift'][0] - immediate['value'] = ( - int(immediate['base_immediate']['value'], 0) << int(immediate['shift']['value']) + immediate["shift"] = immediate["shift"][0] + immediate["value"] = int(immediate["base_immediate"]["value"], 0) << int( + immediate["shift"]["value"] ) return AttrDict({self.IMMEDIATE_ID: immediate}) - if 'float' in immediate: - dict_name = 'float' - if 'double' in immediate: - dict_name = 'double' - if 'exponent' in immediate[dict_name]: + if "float" in immediate: + dict_name = "float" + if "double" in immediate: + dict_name = "double" + if "exponent" in immediate[dict_name]: # nothing to do return AttrDict({self.IMMEDIATE_ID: immediate}) else: # change 'mantissa' key to 'value' return AttrDict( - {self.IMMEDIATE_ID: AttrDict({'value': immediate[dict_name]['mantissa']})} + {self.IMMEDIATE_ID: AttrDict({"value": immediate[dict_name]["mantissa"]})} ) def process_label(self, label): """Post-process label asm line""" # remove duplicated 'name' level due to identifier - label['name'] = label['name']['name'] + label["name"] = label["name"]["name"] return AttrDict({self.LABEL_ID: label}) def process_identifier(self, identifier): """Post-process identifier operand""" # remove value if it consists of symbol+offset - if 'value' in identifier: - del identifier['value'] + if "value" in identifier: + del identifier["value"] return AttrDict({self.IDENTIFIER_ID: identifier}) def get_full_reg_name(self, register): """Return one register name string including all attributes""" - if 'lanes' in register: + if "lanes" in register: return ( - register['prefix'] - + str(register['name']) - + '.' - + str(register['lanes']) - + register['shape'] + register["prefix"] + + str(register["name"]) + + "." + + str(register["lanes"]) + + register["shape"] ) - return register['prefix'] + str(register['name']) + return register["prefix"] + str(register["name"]) def normalize_imd(self, imd): """Normalize immediate to decimal based representation""" - if 'value' in imd: - if imd['value'].lower().startswith('0x'): + if "value" in imd: + if imd["value"].lower().startswith("0x"): # hex, return decimal - return int(imd['value'], 16) - return int(imd['value'], 10) - elif 'float' in imd: - return self.ieee_to_float(imd['float']) - elif 'double' in imd: - return self.ieee_to_float(imd['double']) + return int(imd["value"], 16) + return int(imd["value"], 10) + elif "float" in imd: + return self.ieee_to_float(imd["float"]) + elif "double" in imd: + return self.ieee_to_float(imd["double"]) # identifier return imd def ieee_to_float(self, ieee_val): """Convert IEEE representation to python float""" - exponent = int(ieee_val['exponent'], 10) - if ieee_val['e_sign'] == '-': + exponent = int(ieee_val["exponent"], 10) + if ieee_val["e_sign"] == "-": exponent *= -1 - return float(ieee_val['mantissa']) * (10 ** exponent) + return float(ieee_val["mantissa"]) * (10 ** exponent) def parse_register(self, register_string): raise NotImplementedError def is_gpr(self, register): """Check if register is a general purpose register""" - if register['prefix'] in 'wx': + if register["prefix"] in "wx": return True return False def is_vector_register(self, register): """Check if register is a vector register""" - if register['prefix'] in 'bhsdqvz': + if register["prefix"] in "bhsdqvz": return True return False @@ -510,15 +510,15 @@ class ParserAArch64(BaseParser): def is_reg_dependend_of(self, reg_a, reg_b): """Check if ``reg_a`` is dependent on ``reg_b``""" - prefixes_gpr = 'wx' - prefixes_vec = 'bhsdqvz' - if reg_a['name'] == reg_b['name']: - if reg_a['prefix'].lower() in prefixes_gpr and reg_b['prefix'].lower() in prefixes_gpr: + prefixes_gpr = "wx" + prefixes_vec = "bhsdqvz" + if reg_a["name"] == reg_b["name"]: + if reg_a["prefix"].lower() in prefixes_gpr and reg_b["prefix"].lower() in prefixes_gpr: return True - if reg_a['prefix'].lower() in prefixes_vec and reg_b['prefix'].lower() in prefixes_vec: + if reg_a["prefix"].lower() in prefixes_vec and reg_b["prefix"].lower() in prefixes_vec: return True return False def get_reg_type(self, register): """Get register type""" - return register['prefix'] + return register["prefix"] diff --git a/osaca/parser/parser_x86att.py b/osaca/parser/parser_x86att.py index 8b5b1e7..2b960c6 100755 --- a/osaca/parser/parser_x86att.py +++ b/osaca/parser/parser_x86att.py @@ -19,67 +19,70 @@ class ParserX86ATT(BaseParser): def __init__(self): super().__init__() - self.isa = 'x86' + self.isa = "x86" def construct_parser(self): """Create parser for ARM AArch64 ISA.""" - decimal_number = pp.Combine( - pp.Optional(pp.Literal('-')) + pp.Word(pp.nums) - ).setResultsName('value') - hex_number = pp.Combine(pp.Optional(pp.Literal('-')) + pp.Literal('0x') + pp.Word(pp.hexnums)).setResultsName('value') + decimal_number = pp.Combine(pp.Optional(pp.Literal("-")) + pp.Word(pp.nums)).setResultsName( + "value" + ) + hex_number = pp.Combine( + pp.Optional(pp.Literal("-")) + pp.Literal("0x") + pp.Word(pp.hexnums) + ).setResultsName("value") # Comment - either '#' or '//' (icc) - self.comment = (pp.Literal('#') | pp.Literal('//')) + pp.Group( + self.comment = (pp.Literal("#") | pp.Literal("//")) + pp.Group( pp.ZeroOrMore(pp.Word(pp.printables)) ).setResultsName(self.COMMENT_ID) # Define x86 assembly identifier - relocation = pp.Combine(pp.Literal('@') + pp.Word(pp.alphas)) - id_offset = pp.Word(pp.nums) + pp.Suppress(pp.Literal('+')) - first = pp.Word(pp.alphas + '_.', exact=1) - rest = pp.Word(pp.alphanums + '$_.+-') + relocation = pp.Combine(pp.Literal("@") + pp.Word(pp.alphas)) + id_offset = pp.Word(pp.nums) + pp.Suppress(pp.Literal("+")) + first = pp.Word(pp.alphas + "_.", exact=1) + rest = pp.Word(pp.alphanums + "$_.+-") identifier = pp.Group( - pp.Optional(id_offset).setResultsName('offset') - + pp.Combine(pp.delimitedList( - pp.Combine(first + pp.Optional(rest)), delim='::'), joinString='::' - ).setResultsName('name') - + pp.Optional(relocation).setResultsName('relocation') - ).setResultsName('identifier') + pp.Optional(id_offset).setResultsName("offset") + + pp.Combine( + pp.delimitedList(pp.Combine(first + pp.Optional(rest)), delim="::"), joinString="::" + ).setResultsName("name") + + pp.Optional(relocation).setResultsName("relocation") + ).setResultsName("identifier") # Label - label_rest = pp.Word(pp.alphanums + '$_.+-()') + label_rest = pp.Word(pp.alphanums + "$_.+-()") label_identifier = pp.Group( - pp.Optional(id_offset).setResultsName('offset') - + pp.Combine(pp.delimitedList( - pp.Combine(first + pp.Optional(label_rest)), delim='::'), joinString='::' - ).setResultsName('name') - + pp.Optional(relocation).setResultsName('relocation') - ).setResultsName('identifier') + pp.Optional(id_offset).setResultsName("offset") + + pp.Combine( + pp.delimitedList(pp.Combine(first + pp.Optional(label_rest)), delim="::"), + joinString="::", + ).setResultsName("name") + + pp.Optional(relocation).setResultsName("relocation") + ).setResultsName("identifier") numeric_identifier = pp.Group( - pp.Word(pp.nums).setResultsName('name') - + pp.Optional(pp.oneOf('b f', caseless=True).setResultsName('suffix')) - ).setResultsName('identifier') + pp.Word(pp.nums).setResultsName("name") + + pp.Optional(pp.oneOf("b f", caseless=True).setResultsName("suffix")) + ).setResultsName("identifier") self.label = pp.Group( - (label_identifier | numeric_identifier).setResultsName('name') - + pp.Literal(':') + (label_identifier | numeric_identifier).setResultsName("name") + + pp.Literal(":") + pp.Optional(self.comment) ).setResultsName(self.LABEL_ID) # Register: pp.Regex('^%[0-9a-zA-Z]+{}{z},?') self.register = pp.Group( - pp.Literal('%') - + pp.Word(pp.alphanums).setResultsName('name') - + pp.Optional(pp.Literal('(') + pp.Word(pp.nums) + pp.Literal(')')) + pp.Literal("%") + + pp.Word(pp.alphanums).setResultsName("name") + + pp.Optional(pp.Literal("(") + pp.Word(pp.nums) + pp.Literal(")")) + pp.Optional( - pp.Literal('{') - + pp.Optional(pp.Suppress(pp.Literal('%'))) - + pp.Word(pp.alphanums).setResultsName('mask') - + pp.Literal('}') + pp.Literal("{") + + pp.Optional(pp.Suppress(pp.Literal("%"))) + + pp.Word(pp.alphanums).setResultsName("mask") + + pp.Literal("}") + pp.Optional( - pp.Suppress(pp.Literal('{')) - + pp.Literal('z').setResultsName('zeroing') - + pp.Suppress(pp.Literal('}')) + pp.Suppress(pp.Literal("{")) + + pp.Literal("z").setResultsName("zeroing") + + pp.Suppress(pp.Literal("}")) ) ) ).setResultsName(self.REGISTER_ID) # Immediate: pp.Regex('^\$(-?[0-9]+)|(0x[0-9a-fA-F]+),?') - symbol_immediate = '$' + symbol_immediate = "$" immediate = pp.Group( pp.Literal(symbol_immediate) + (hex_number | decimal_number | identifier) ).setResultsName(self.IMMEDIATE_ID) @@ -88,53 +91,52 @@ class ParserX86ATT(BaseParser): offset = pp.Group(identifier | hex_number | decimal_number).setResultsName( self.IMMEDIATE_ID ) - scale = pp.Word('1248', exact=1) + scale = pp.Word("1248", exact=1) # Segment register extension segment_extension = ( hex_number ^ pp.Word(pp.nums) ^ pp.Group( - pp.Optional(offset.setResultsName('offset')) - + pp.Literal('(') - + pp.Optional(self.register.setResultsName('base')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(self.register.setResultsName('index')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(scale.setResultsName('scale')) - + pp.Literal(')') + pp.Optional(offset.setResultsName("offset")) + + pp.Literal("(") + + pp.Optional(self.register.setResultsName("base")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(self.register.setResultsName("index")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(scale.setResultsName("scale")) + + pp.Literal(")") ) ) memory_segmentation = ( - self.register.setResultsName('base') - + pp.Literal(':') + self.register.setResultsName("base") + + pp.Literal(":") + segment_extension.setResultsName(self.SEGMENT_EXT_ID) ) # Memory: offset | seg:seg_ext | offset(base, index, scale){mask} - memory_abs = ( - pp.Suppress(pp.Literal('*')) - + (offset | self.register).setResultsName('offset') + memory_abs = pp.Suppress(pp.Literal("*")) + (offset | self.register).setResultsName( + "offset" ) memory = pp.Group( ( - pp.Optional(pp.Suppress(pp.Literal('*'))) - + pp.Optional(offset.setResultsName('offset')) - + pp.Literal('(') - + pp.Optional(self.register.setResultsName('base')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(self.register.setResultsName('index')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(scale.setResultsName('scale')) - + pp.Literal(')') + pp.Optional(pp.Suppress(pp.Literal("*"))) + + pp.Optional(offset.setResultsName("offset")) + + pp.Literal("(") + + pp.Optional(self.register.setResultsName("base")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(self.register.setResultsName("index")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(scale.setResultsName("scale")) + + pp.Literal(")") + pp.Optional( - pp.Literal('{') - + pp.Optional(pp.Suppress(pp.Literal('%'))) - + pp.Word(pp.alphanums).setResultsName('mask') - + pp.Literal('}') + pp.Literal("{") + + pp.Optional(pp.Suppress(pp.Literal("%"))) + + pp.Word(pp.alphanums).setResultsName("mask") + + pp.Literal("}") ) ) | memory_abs | memory_segmentation - | (hex_number | pp.Word(pp.nums)).setResultsName('offset') + | (hex_number | pp.Word(pp.nums)).setResultsName("offset") ).setResultsName(self.MEMORY_ID) # Directive @@ -143,23 +145,23 @@ class ParserX86ATT(BaseParser): directive_parameter = ( pp.quotedString ^ ( - pp.Word(pp.printables, excludeChars=',#') - + pp.Optional(pp.Suppress(pp.Literal(','))) + pp.Word(pp.printables, excludeChars=",#") + + pp.Optional(pp.Suppress(pp.Literal(","))) ) - ^ pp.Suppress(pp.Literal(',')) + ^ pp.Suppress(pp.Literal(",")) ) self.directive = pp.Group( - pp.Literal('.') - + pp.Word(pp.alphanums + '_').setResultsName('name') - + pp.ZeroOrMore(directive_parameter).setResultsName('parameters') + pp.Literal(".") + + pp.Word(pp.alphanums + "_").setResultsName("name") + + pp.ZeroOrMore(directive_parameter).setResultsName("parameters") + pp.Optional(self.comment) ).setResultsName(self.DIRECTIVE_ID) # Instructions # Mnemonic - mnemonic = pp.ZeroOrMore(pp.Literal('data16') | pp.Literal('data32')) + pp.Word( - pp.alphanums + ',' - ).setResultsName('mnemonic') + mnemonic = pp.ZeroOrMore(pp.Literal("data16") | pp.Literal("data32")) + pp.Word( + pp.alphanums + "," + ).setResultsName("mnemonic") # Combine to instruction form operand_first = pp.Group( self.register ^ immediate ^ memory ^ identifier ^ numeric_identifier @@ -167,13 +169,13 @@ class ParserX86ATT(BaseParser): operand_rest = pp.Group(self.register ^ immediate ^ memory) self.instruction_parser = ( mnemonic - + pp.Optional(operand_first.setResultsName('operand1')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand2')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand3')) - + pp.Optional(pp.Suppress(pp.Literal(','))) - + pp.Optional(operand_rest.setResultsName('operand4')) + + pp.Optional(operand_first.setResultsName("operand1")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand2")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand3")) + + pp.Optional(pp.Suppress(pp.Literal(","))) + + pp.Optional(operand_rest.setResultsName("operand4")) + pp.Optional(self.comment) ) @@ -202,8 +204,8 @@ class ParserX86ATT(BaseParser): self.DIRECTIVE_ID: None, self.COMMENT_ID: None, self.LABEL_ID: None, - 'line': line, - 'line_number': line_number, + "line": line, + "line_number": line_number, } ) result = None @@ -212,7 +214,7 @@ class ParserX86ATT(BaseParser): try: result = self.process_operand(self.comment.parseString(line, parseAll=True).asDict()) result = AttrDict.convert_dict(result) - instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID]) + instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID]) except pp.ParseException: pass @@ -221,9 +223,9 @@ class ParserX86ATT(BaseParser): try: result = self.process_operand(self.label.parseString(line, parseAll=True).asDict()) result = AttrDict.convert_dict(result) - instruction_form[self.LABEL_ID] = result[self.LABEL_ID]['name'] + instruction_form[self.LABEL_ID] = result[self.LABEL_ID]["name"] if self.COMMENT_ID in result[self.LABEL_ID]: - instruction_form[self.COMMENT_ID] = ' '.join( + instruction_form[self.COMMENT_ID] = " ".join( result[self.LABEL_ID][self.COMMENT_ID] ) except pp.ParseException: @@ -238,12 +240,12 @@ class ParserX86ATT(BaseParser): result = AttrDict.convert_dict(result) instruction_form[self.DIRECTIVE_ID] = AttrDict( { - 'name': result[self.DIRECTIVE_ID]['name'], - 'parameters': result[self.DIRECTIVE_ID]['parameters'], + "name": result[self.DIRECTIVE_ID]["name"], + "parameters": result[self.DIRECTIVE_ID]["parameters"], } ) if self.COMMENT_ID in result[self.DIRECTIVE_ID]: - instruction_form[self.COMMENT_ID] = ' '.join( + instruction_form[self.COMMENT_ID] = " ".join( result[self.DIRECTIVE_ID][self.COMMENT_ID] ) except pp.ParseException: @@ -255,7 +257,7 @@ class ParserX86ATT(BaseParser): result = self.parse_instruction(line) except pp.ParseException: raise ValueError( - 'Could not parse instruction on line {}: {!r}'.format(line_number, line) + "Could not parse instruction on line {}: {!r}".format(line_number, line) ) instruction_form[self.INSTRUCTION_ID] = result[self.INSTRUCTION_ID] instruction_form[self.OPERANDS_ID] = result[self.OPERANDS_ID] @@ -275,22 +277,22 @@ class ParserX86ATT(BaseParser): operands = [] # Add operands to list # Check first operand - if 'operand1' in result: - operands.append(self.process_operand(result['operand1'])) + if "operand1" in result: + operands.append(self.process_operand(result["operand1"])) # Check second operand - if 'operand2' in result: - operands.append(self.process_operand(result['operand2'])) + if "operand2" in result: + operands.append(self.process_operand(result["operand2"])) # Check third operand - if 'operand3' in result: - operands.append(self.process_operand(result['operand3'])) + if "operand3" in result: + operands.append(self.process_operand(result["operand3"])) # Check fourth operand - if 'operand4' in result: - operands.append(self.process_operand(result['operand4'])) + if "operand4" in result: + operands.append(self.process_operand(result["operand4"])) return_dict = AttrDict( { - self.INSTRUCTION_ID: result['mnemonic'].split(',')[0], + self.INSTRUCTION_ID: result["mnemonic"].split(",")[0], self.OPERANDS_ID: operands, - self.COMMENT_ID: ' '.join(result[self.COMMENT_ID]) + self.COMMENT_ID: " ".join(result[self.COMMENT_ID]) if self.COMMENT_ID in result else None, } @@ -311,23 +313,23 @@ class ParserX86ATT(BaseParser): return operand def process_directive(self, directive): - directive_new = {'name': directive['name'], 'parameters': []} - if 'parameters' in directive: - directive_new['parameters'] = directive['parameters'] - if 'comment' in directive: - directive_new['comment'] = directive['comment'] + directive_new = {"name": directive["name"], "parameters": []} + if "parameters" in directive: + directive_new["parameters"] = directive["parameters"] + if "comment" in directive: + directive_new["comment"] = directive["comment"] return AttrDict({self.DIRECTIVE_ID: directive_new}) def process_memory_address(self, memory_address): """Post-process memory address operand""" # Remove unecessarily created dictionary entries during memory address parsing - offset = memory_address.get('offset', None) - base = memory_address.get('base', None) - index = memory_address.get('index', None) - scale = 1 if 'scale' not in memory_address else int(memory_address['scale']) + offset = memory_address.get("offset", None) + base = memory_address.get("base", None) + index = memory_address.get("index", None) + scale = 1 if "scale" not in memory_address else int(memory_address["scale"]) if isinstance(offset, str) and base is None and index is None: - offset = {'value': offset} - new_dict = AttrDict({'offset': offset, 'base': base, 'index': index, 'scale': scale}) + offset = {"value": offset} + new_dict = AttrDict({"offset": offset, "base": base, "index": index, "scale": scale}) # Add segmentation extension if existing if self.SEGMENT_EXT_ID in memory_address: new_dict[self.SEGMENT_EXT_ID] = memory_address[self.SEGMENT_EXT_ID] @@ -336,12 +338,12 @@ class ParserX86ATT(BaseParser): def process_label(self, label): """Post-process label asm line""" # remove duplicated 'name' level due to identifier - label['name'] = label['name'][0]['name'] + label["name"] = label["name"][0]["name"] return AttrDict({self.LABEL_ID: label}) def process_immediate(self, immediate): """Post-process immediate operand""" - if 'identifier' in immediate: + if "identifier" in immediate: # actually an identifier, change declaration return immediate # otherwise nothing to do @@ -350,15 +352,15 @@ class ParserX86ATT(BaseParser): def get_full_reg_name(self, register): """Return one register name string including all attributes""" # nothing to do - return register['name'] + return register["name"] def normalize_imd(self, imd): """Normalize immediate to decimal based representation""" - if 'value' in imd: - if imd['value'].lower().startswith('0x'): + if "value" in imd: + if imd["value"].lower().startswith("0x"): # hex, return decimal - return int(imd['value'], 16) - return int(imd['value'], 10) + return int(imd["value"], 16) + return int(imd["value"], 10) # identifier return imd @@ -373,8 +375,8 @@ class ParserX86ATT(BaseParser): def is_reg_dependend_of(self, reg_a, reg_b): """Check if ``reg_a`` is dependent on ``reg_b``""" # Normalize name - reg_a_name = reg_a['name'].upper() - reg_b_name = reg_b['name'].upper() + reg_a_name = reg_a["name"].upper() + reg_b_name = reg_b["name"].upper() # Check if they are the same registers if reg_a_name == reg_b_name: @@ -388,13 +390,13 @@ class ParserX86ATT(BaseParser): return False # Check basic GPRs gpr_groups = { - 'A': ['RAX', 'EAX', 'AX', 'AH', 'AL'], - 'B': ['RBX', 'EBX', 'BX', 'BH', 'BL'], - 'C': ['RCX', 'ECX', 'CX', 'CH', 'CL'], - 'D': ['RDX', 'EDX', 'DX', 'DH', 'DL'], - 'SP': ['RSP', 'ESP', 'SP', 'SPL'], - 'SRC': ['RSI', 'ESI', 'SI', 'SIL'], - 'DST': ['RDI', 'EDI', 'DI', 'DIL'] + "A": ["RAX", "EAX", "AX", "AH", "AL"], + "B": ["RBX", "EBX", "BX", "BH", "BL"], + "C": ["RCX", "ECX", "CX", "CH", "CL"], + "D": ["RDX", "EDX", "DX", "DH", "DL"], + "SP": ["RSP", "ESP", "SP", "SPL"], + "SRC": ["RSI", "ESI", "SI", "SIL"], + "DST": ["RDI", "EDI", "DI", "DIL"], } if self.is_basic_gpr(reg_a): if self.is_basic_gpr(reg_b): @@ -405,8 +407,8 @@ class ParserX86ATT(BaseParser): return False # Check other GPRs - ma = re.match(r'R([0-9]+)[DWB]?', reg_a_name) - mb = re.match(r'R([0-9]+)[DWB]?', reg_b_name) + ma = re.match(r"R([0-9]+)[DWB]?", reg_a_name) + mb = re.match(r"R([0-9]+)[DWB]?", reg_b_name) if ma and mb and ma.group(1) == mb.group(1): return True @@ -415,9 +417,8 @@ class ParserX86ATT(BaseParser): def is_basic_gpr(self, register): """Check if register is a basic general purpose register (ebi, rax, ...)""" - if ( - any(char.isdigit() for char in register['name']) - or any(register['name'].lower().startswith(x) for x in ['mm', 'xmm', 'ymm', 'zmm']) + if any(char.isdigit() for char in register["name"]) or any( + register["name"].lower().startswith(x) for x in ["mm", "xmm", "ymm", "zmm"] ): return False return True @@ -428,13 +429,13 @@ class ParserX86ATT(BaseParser): return False if self.is_basic_gpr(register): return True - return re.match(r'R([0-9]+)[DWB]?', register['name'], re.IGNORECASE) + return re.match(r"R([0-9]+)[DWB]?", register["name"], re.IGNORECASE) def is_vector_register(self, register): """Check if register is a vector register""" if register is None: return False - if register['name'].rstrip(string.digits).lower() in ['mm', 'xmm', 'ymm', 'zmm']: + if register["name"].rstrip(string.digits).lower() in ["mm", "xmm", "ymm", "zmm"]: return True return False @@ -443,7 +444,7 @@ class ParserX86ATT(BaseParser): if register is None: return False if self.is_gpr(register): - return 'gpr' + return "gpr" elif self.is_vector_register(register): - return register['name'].rstrip(string.digits).lower() + return register["name"].rstrip(string.digits).lower() raise ValueError diff --git a/osaca/semantics/__init__.py b/osaca/semantics/__init__.py index f029dd0..8a0b000 100644 --- a/osaca/semantics/__init__.py +++ b/osaca/semantics/__init__.py @@ -11,13 +11,13 @@ from .marker_utils import reduce_to_section, find_basic_blocks, find_basic_loop_ from .marker_utils import find_jump_labels __all__ = [ - 'MachineModel', - 'KernelDG', - 'reduce_to_section', - 'ArchSemantics', - 'ISASemantics', - 'INSTR_FLAGS', - 'find_basic_blocks', - 'find_basic_loop_bodies', - 'find_jump_labels', + "MachineModel", + "KernelDG", + "reduce_to_section", + "ArchSemantics", + "ISASemantics", + "INSTR_FLAGS", + "find_basic_blocks", + "find_basic_loop_bodies", + "find_jump_labels", ] diff --git a/osaca/semantics/arch_semantics.py b/osaca/semantics/arch_semantics.py index c72a3d7..389eaa1 100755 --- a/osaca/semantics/arch_semantics.py +++ b/osaca/semantics/arch_semantics.py @@ -11,7 +11,7 @@ from .isa_semantics import INSTR_FLAGS, ISASemantics class ArchSemantics(ISASemantics): - GAS_SUFFIXES = 'bswlqt' + GAS_SUFFIXES = "bswlqt" def __init__(self, machine_model: MachineModel, path_to_yaml=None): super().__init__(machine_model.get_ISA().lower(), path_to_yaml=path_to_yaml) @@ -42,19 +42,17 @@ class ArchSemantics(ISASemantics): kernel.reverse() port_list = self._machine_model.get_ports() for instruction_form in kernel: - for uop in instruction_form['port_uops']: + for uop in instruction_form["port_uops"]: cycles = uop[0] ports = list(uop[1]) indices = [port_list.index(p) for p in ports] # check if port sum of used ports for uop are unbalanced port_sums = self._to_list(itemgetter(*indices)(self.get_throughput_sum(kernel))) - instr_ports = self._to_list( - itemgetter(*indices)(instruction_form['port_pressure']) - ) + instr_ports = self._to_list(itemgetter(*indices)(instruction_form["port_pressure"])) if len(set(port_sums)) > 1: # balance ports # init list for keeping track of the current change - differences = [cycles / len(ports) for p in ports] + differences = [cycles / len(ports) for p in ports] for _ in range(int(cycles * (1 / INC))): if len(instr_ports) == 1: # no balancing possible anymore @@ -66,7 +64,7 @@ class ArchSemantics(ISASemantics): differences[max_port_idx] -= INC differences[min_port_idx] += INC # instr_ports = [round(p, 2) for p in instr_ports] - self._itemsetter(*indices)(instruction_form['port_pressure'], *instr_ports) + self._itemsetter(*indices)(instruction_form["port_pressure"], *instr_ports) # check if min port is zero if round(min(instr_ports), 2) <= 0: # if port_pressure is not exactly 0.00, add the residual to @@ -79,22 +77,22 @@ class ArchSemantics(ISASemantics): # delete it del differences[instr_ports.index(min(instr_ports))] self._itemsetter(*indices)( - instruction_form['port_pressure'], *instr_ports + instruction_form["port_pressure"], *instr_ports ) zero_index = [ p for p in indices - if round(instruction_form['port_pressure'][p], 2) == 0 + if round(instruction_form["port_pressure"][p], 2) == 0 ][0] - instruction_form['port_pressure'][zero_index] = 0.0 + instruction_form["port_pressure"][zero_index] = 0.0 # Remove from further balancing indices = [ - p for p in indices if instruction_form['port_pressure'][p] > 0 + p for p in indices if instruction_form["port_pressure"][p] > 0 ] instr_ports = self._to_list( - itemgetter(*indices)(instruction_form['port_pressure']) + itemgetter(*indices)(instruction_form["port_pressure"]) ) - # never remove more than the fixed utilization per uop and port, i.e., + # never remove more than the fixed utilization per uop and port, i.e., # cycles/len(ports) if round(min(differences), 2) <= 0: # don't worry if port_pressure isn't exactly 0 and just @@ -102,7 +100,7 @@ class ArchSemantics(ISASemantics): # pressure is not 0 del indices[differences.index(min(differences))] instr_ports = self._to_list( - itemgetter(*indices)(instruction_form['port_pressure']) + itemgetter(*indices)(instruction_form["port_pressure"]) ) del differences[differences.index(min(differences))] port_sums = self._to_list( @@ -112,14 +110,14 @@ class ArchSemantics(ISASemantics): def set_hidden_loads(self, kernel): """Hide loads behind stores if architecture supports hidden loads (depricated)""" - loads = [instr for instr in kernel if INSTR_FLAGS.HAS_LD in instr['flags']] - stores = [instr for instr in kernel if INSTR_FLAGS.HAS_ST in instr['flags']] + loads = [instr for instr in kernel if INSTR_FLAGS.HAS_LD in instr["flags"]] + stores = [instr for instr in kernel if INSTR_FLAGS.HAS_ST in instr["flags"]] # Filter instructions including load and store - load_ids = [instr['line_number'] for instr in loads] - store_ids = [instr['line_number'] for instr in stores] + load_ids = [instr["line_number"] for instr in loads] + store_ids = [instr["line_number"] for instr in stores] shared_ldst = list(set(load_ids).intersection(set(store_ids))) - loads = [instr for instr in loads if instr['line_number'] not in shared_ldst] - stores = [instr for instr in stores if instr['line_number'] not in shared_ldst] + loads = [instr for instr in loads if instr["line_number"] not in shared_ldst] + stores = [instr for instr in stores if instr["line_number"] not in shared_ldst] if len(stores) == 0 or len(loads) == 0: # nothing to do @@ -127,53 +125,53 @@ class ArchSemantics(ISASemantics): if len(loads) <= len(stores): # Hide all loads for load in loads: - load['flags'] += [INSTR_FLAGS.HIDDEN_LD] - load['port_pressure'] = self._nullify_data_ports(load['port_pressure']) + load["flags"] += [INSTR_FLAGS.HIDDEN_LD] + load["port_pressure"] = self._nullify_data_ports(load["port_pressure"]) else: for store in stores: # Get 'closest' load instruction min_distance_load = min( [ ( - abs(load_instr['line_number'] - store['line_number']), - load_instr['line_number'], + abs(load_instr["line_number"] - store["line_number"]), + load_instr["line_number"], ) for load_instr in loads - if INSTR_FLAGS.HIDDEN_LD not in load_instr['flags'] + if INSTR_FLAGS.HIDDEN_LD not in load_instr["flags"] ] ) - load = [instr for instr in kernel if instr['line_number'] == min_distance_load[1]][ + load = [instr for instr in kernel if instr["line_number"] == min_distance_load[1]][ 0 ] # Hide load - load['flags'] += [INSTR_FLAGS.HIDDEN_LD] - load['port_pressure'] = self._nullify_data_ports(load['port_pressure']) + load["flags"] += [INSTR_FLAGS.HIDDEN_LD] + load["port_pressure"] = self._nullify_data_ports(load["port_pressure"]) # get parser result and assign throughput and latency value to instruction form # mark instruction form with semantic flags def assign_tp_lt(self, instruction_form): """Assign throughput and latency to an instruction form.""" flags = [] - port_number = len(self._machine_model['ports']) - if instruction_form['instruction'] is None: + port_number = len(self._machine_model["ports"]) + if instruction_form["instruction"] is None: # No instruction (label, comment, ...) --> ignore throughput = 0.0 latency = 0.0 latency_wo_load = latency - instruction_form['port_pressure'] = [0.0 for i in range(port_number)] - instruction_form['port_uops'] = [] + instruction_form["port_pressure"] = [0.0 for i in range(port_number)] + instruction_form["port_uops"] = [] else: instruction_data = self._machine_model.get_instruction( - instruction_form['instruction'], instruction_form['operands'] + instruction_form["instruction"], instruction_form["operands"] ) if ( not instruction_data - and self._isa == 'x86' - and instruction_form['instruction'][-1] in self.GAS_SUFFIXES + and self._isa == "x86" + and instruction_form["instruction"][-1] in self.GAS_SUFFIXES ): # check for instruction without GAS suffix instruction_data = self._machine_model.get_instruction( - instruction_form['instruction'][:-1], instruction_form['operands'] + instruction_form["instruction"][:-1], instruction_form["operands"] ) if instruction_data: # instruction form in DB @@ -190,79 +188,79 @@ class ArchSemantics(ISASemantics): assign_unknown = True # check for equivalent register-operands DB entry if LD if ( - INSTR_FLAGS.HAS_LD in instruction_form['flags'] - or INSTR_FLAGS.HAS_ST in instruction_form['flags'] + INSTR_FLAGS.HAS_LD in instruction_form["flags"] + or INSTR_FLAGS.HAS_ST in instruction_form["flags"] ): # dynamically combine LD/ST and reg form of instruction form # substitute mem and look for reg-only variant - operands = self.substitute_mem_address(instruction_form['operands']) + operands = self.substitute_mem_address(instruction_form["operands"]) instruction_data_reg = self._machine_model.get_instruction( - instruction_form['instruction'], operands + instruction_form["instruction"], operands ) if ( not instruction_data_reg - and self._isa == 'x86' - and instruction_form['instruction'][-1] in self.GAS_SUFFIXES + and self._isa == "x86" + and instruction_form["instruction"][-1] in self.GAS_SUFFIXES ): # check for instruction without GAS suffix instruction_data_reg = self._machine_model.get_instruction( - instruction_form['instruction'][:-1], operands + instruction_form["instruction"][:-1], operands ) if instruction_data_reg: assign_unknown = False reg_type = self._parser.get_reg_type( - instruction_data_reg['operands'][ + instruction_data_reg["operands"][ operands.index(self._create_reg_wildcard()) ] ) data_port_pressure = [0.0 for _ in range(port_number)] data_port_uops = [] - if INSTR_FLAGS.HAS_LD in instruction_form['flags']: + if INSTR_FLAGS.HAS_LD in instruction_form["flags"]: # LOAD performance data data_port_uops = self._machine_model.get_load_throughput( [ - x['memory'] - for x in instruction_form['semantic_operands']['source'] - + instruction_form['semantic_operands']['src_dst'] - if 'memory' in x + x["memory"] + for x in instruction_form["semantic_operands"]["source"] + + instruction_form["semantic_operands"]["src_dst"] + if "memory" in x ][0] ) data_port_pressure = self._machine_model.average_port_pressure( data_port_uops ) - if 'load_throughput_multiplier' in self._machine_model: - multiplier = self._machine_model['load_throughput_multiplier'][ + if "load_throughput_multiplier" in self._machine_model: + multiplier = self._machine_model["load_throughput_multiplier"][ reg_type ] data_port_pressure = [pp * multiplier for pp in data_port_pressure] - if INSTR_FLAGS.HAS_ST in instruction_form['flags']: + if INSTR_FLAGS.HAS_ST in instruction_form["flags"]: # STORE performance data destinations = ( - instruction_form['semantic_operands']['destination'] - + instruction_form['semantic_operands']['src_dst'] + instruction_form["semantic_operands"]["destination"] + + instruction_form["semantic_operands"]["src_dst"] ) st_data_port_uops = self._machine_model.get_store_throughput( - [x['memory'] for x in destinations if 'memory' in x][0] + [x["memory"] for x in destinations if "memory" in x][0] ) # zero data port pressure and remove HAS_ST flag if # - no mem operand in dst && # - all mem operands in src_dst are pre-/post-indexed # since it is no mem store if ( - self._isa == 'aarch64' - and 'memory' - not in instruction_form['semantic_operands']['destination'] + self._isa == "aarch64" + and "memory" + not in instruction_form["semantic_operands"]["destination"] and all( [ - 'post_indexed' in op['memory'] - or 'pre_indexed' in op['memory'] - for op in instruction_form['semantic_operands']['src_dst'] - if 'memory' in op + "post_indexed" in op["memory"] + or "pre_indexed" in op["memory"] + for op in instruction_form["semantic_operands"]["src_dst"] + if "memory" in op ] ) ): st_data_port_uops = [] - instruction_form['flags'].remove(INSTR_FLAGS.HAS_ST) + instruction_form["flags"].remove(INSTR_FLAGS.HAS_ST) # sum up all data ports in case for LOAD and STORE st_data_port_pressure = self._machine_model.average_port_pressure( @@ -273,21 +271,21 @@ class ArchSemantics(ISASemantics): ] data_port_uops += st_data_port_uops throughput = max( - max(data_port_pressure), instruction_data_reg['throughput'] + max(data_port_pressure), instruction_data_reg["throughput"] ) - latency = instruction_data_reg['latency'] + latency = instruction_data_reg["latency"] # Add LD and ST latency latency += ( self._machine_model.get_load_latency(reg_type) - if INSTR_FLAGS.HAS_LD in instruction_form['flags'] + if INSTR_FLAGS.HAS_LD in instruction_form["flags"] else 0 ) latency += ( self._machine_model.get_store_latency(reg_type) - if INSTR_FLAGS.HAS_ST in instruction_form['flags'] + if INSTR_FLAGS.HAS_ST in instruction_form["flags"] else 0 ) - latency_wo_load = instruction_data_reg['latency'] + latency_wo_load = instruction_data_reg["latency"] # add latency of ADD if post- or pre-indexed load # TODO more investigation: check dot-graph, wrong latency distribution! # if ( @@ -302,17 +300,17 @@ class ArchSemantics(ISASemantics): # ) # ): # latency_wo_load = 1.0 - instruction_form['port_pressure'] = [ + instruction_form["port_pressure"] = [ sum(x) for x in zip( data_port_pressure, self._machine_model.average_port_pressure( - instruction_data_reg['port_pressure'] + instruction_data_reg["port_pressure"] ), ) ] - instruction_form['port_uops'] = list( - chain(instruction_data_reg['port_pressure'], data_port_uops) + instruction_form["port_uops"] = list( + chain(instruction_data_reg["port_pressure"], data_port_uops) ) if assign_unknown: @@ -320,68 +318,66 @@ class ArchSemantics(ISASemantics): throughput = 0.0 latency = 0.0 latency_wo_load = latency - instruction_form['port_pressure'] = [0.0 for i in range(port_number)] - instruction_form['port_uops'] = [] + instruction_form["port_pressure"] = [0.0 for i in range(port_number)] + instruction_form["port_uops"] = [] flags += [INSTR_FLAGS.TP_UNKWN, INSTR_FLAGS.LT_UNKWN] # flatten flag list flags = list(set(flags)) - if 'flags' not in instruction_form: - instruction_form['flags'] = flags + if "flags" not in instruction_form: + instruction_form["flags"] = flags else: - instruction_form['flags'] += flags - instruction_form['throughput'] = throughput - instruction_form['latency'] = latency - instruction_form['latency_wo_load'] = latency_wo_load + instruction_form["flags"] += flags + instruction_form["throughput"] = throughput + instruction_form["latency"] = latency + instruction_form["latency_wo_load"] = latency_wo_load # for later CP and loop-carried dependency analysis - instruction_form['latency_cp'] = 0 - instruction_form['latency_lcd'] = 0 + instruction_form["latency_cp"] = 0 + instruction_form["latency_lcd"] = 0 def _handle_instruction_found(self, instruction_data, port_number, instruction_form, flags): """Apply performance data to instruction if it was found in the archDB""" - throughput = instruction_data['throughput'] - port_pressure = self._machine_model.average_port_pressure( - instruction_data['port_pressure'] - ) - instruction_form['port_uops'] = instruction_data['port_pressure'] + throughput = instruction_data["throughput"] + port_pressure = self._machine_model.average_port_pressure(instruction_data["port_pressure"]) + instruction_form["port_uops"] = instruction_data["port_pressure"] try: assert isinstance(port_pressure, list) assert len(port_pressure) == port_number - instruction_form['port_pressure'] = port_pressure + instruction_form["port_pressure"] = port_pressure if sum(port_pressure) == 0 and throughput is not None: # port pressure on all ports 0 --> not bound to a port flags.append(INSTR_FLAGS.NOT_BOUND) except AssertionError: warnings.warn( - 'Port pressure could not be imported correctly from database. ' - + 'Please check entry for:\n {}'.format(instruction_form) + "Port pressure could not be imported correctly from database. " + + "Please check entry for:\n {}".format(instruction_form) ) - instruction_form['port_pressure'] = [0.0 for i in range(port_number)] - instruction_form['port_uops'] = [] + instruction_form["port_pressure"] = [0.0 for i in range(port_number)] + instruction_form["port_uops"] = [] flags.append(INSTR_FLAGS.TP_UNKWN) if throughput is None: # assume 0 cy and mark as unknown throughput = 0.0 flags.append(INSTR_FLAGS.TP_UNKWN) - latency = instruction_data['latency'] + latency = instruction_data["latency"] latency_wo_load = latency if latency is None: # assume 0 cy and mark as unknown latency = 0.0 latency_wo_load = latency flags.append(INSTR_FLAGS.LT_UNKWN) - if INSTR_FLAGS.HAS_LD in instruction_form['flags']: + if INSTR_FLAGS.HAS_LD in instruction_form["flags"]: flags.append(INSTR_FLAGS.LD) return throughput, port_pressure, latency, latency_wo_load - def convert_op_to_reg(self, reg_type, reg_id='0'): + def convert_op_to_reg(self, reg_type, reg_id="0"): """Create register operand for a memory addressing operand""" - if self._isa == 'x86': - if reg_type == 'gpr': - register = {'register': {'name': 'r' + str(int(reg_id) + 9)}} + if self._isa == "x86": + if reg_type == "gpr": + register = {"register": {"name": "r" + str(int(reg_id) + 9)}} else: - register = {'register': {'name': reg_type + reg_id}} - elif self._isa == 'aarch64': - register = {'register': {'prefix': reg_type, 'name': reg_id}} + register = {"register": {"name": reg_type + reg_id}} + elif self._isa == "aarch64": + register = {"register": {"prefix": reg_type, "name": reg_id}} return register def _nullify_data_ports(self, port_pressure): @@ -398,7 +394,9 @@ class ArchSemantics(ISASemantics): def g(obj, value): obj[item] = value + else: + def g(obj, *values): for item, value in zip(items, values): obj[item] = value @@ -416,7 +414,7 @@ class ArchSemantics(ISASemantics): """Get the overall throughput sum separated by port of all instructions of a kernel.""" # ignoring all lines with throughput == 0.0, because there won't be anything to sum up # typically comment, label and non-instruction lines - port_pressures = [instr['port_pressure'] for instr in kernel if instr['throughput'] != 0.0] + port_pressures = [instr["port_pressure"] for instr in kernel if instr["throughput"] != 0.0] # Essentially summing up each columns of port_pressures, where each column is one port # and each row is one line of the kernel # round is necessary to ensure termination of ArchsSemantics.assign_optimal_throughput diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index 65882e3..ba0bd8e 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -18,76 +18,77 @@ from osaca.parser import ParserX86ATT class MachineModel(object): - WILDCARD = '*' + WILDCARD = "*" INTERNAL_VERSION = 1 # increase whenever self._data format changes to invalidate cache! def __init__(self, arch=None, path_to_yaml=None, isa=None, lazy=False): if not arch and not path_to_yaml: if not isa: - raise ValueError('One of arch, path_to_yaml and isa must be specified') + raise ValueError("One of arch, path_to_yaml and isa must be specified") self._data = { - 'osaca_version': str(__version__), - 'micro_architecture': None, - 'arch_code': None, - 'isa': isa, - 'ROB_size': None, - 'retired_uOps_per_cycle': None, - 'scheduler_size': None, - 'hidden_loads': None, - 'load_latency': {}, - 'load_throughput': [ - {'base': b, 'index': i, 'offset': o, 'scale': s, 'port_pressure': []} - for b, i, o, s in product(['gpr'], ['gpr', None], ['imd', None], [1, 8]) + "osaca_version": str(__version__), + "micro_architecture": None, + "arch_code": None, + "isa": isa, + "ROB_size": None, + "retired_uOps_per_cycle": None, + "scheduler_size": None, + "hidden_loads": None, + "load_latency": {}, + "load_throughput": [ + {"base": b, "index": i, "offset": o, "scale": s, "port_pressure": []} + for b, i, o, s in product(["gpr"], ["gpr", None], ["imd", None], [1, 8]) ], - 'load_throughput_default': [], - 'store_throughput': [], - 'store_throughput_default': [], - 'ports': [], - 'port_model_scheme': None, - 'instruction_forms': [] + "load_throughput_default": [], + "store_throughput": [], + "store_throughput_default": [], + "ports": [], + "port_model_scheme": None, + "instruction_forms": [], } else: if arch and path_to_yaml: - raise ValueError('Only one of arch and path_to_yaml is allowed.') + raise ValueError("Only one of arch and path_to_yaml is allowed.") self._path = path_to_yaml self._arch = arch yaml = self._create_yaml_object() if arch: self._arch = arch.lower() - self._path = utils.find_datafile(self._arch + '.yml') + self._path = utils.find_datafile(self._arch + ".yml") # check if file is cached cached = self._get_cached(self._path) if not lazy else False if cached: self._data = cached else: # otherwise load - with open(self._path, 'r') as f: + with open(self._path, "r") as f: if not lazy: self._data = yaml.load(f) else: - file_content = '' + file_content = "" line = f.readline() - while 'instruction_forms:' not in line: + while "instruction_forms:" not in line: file_content += line line = f.readline() self._data = yaml.load(file_content) - self._data['instruction_forms'] = [] + self._data["instruction_forms"] = [] # separate multi-alias instruction forms - for entry in [x for x in self._data['instruction_forms'] - if isinstance(x['name'], list)]: - for name in entry['name']: - new_entry = {'name': name} - for k in [x for x in entry.keys() if x != 'name']: + for entry in [ + x for x in self._data["instruction_forms"] if isinstance(x["name"], list) + ]: + for name in entry["name"]: + new_entry = {"name": name} + for k in [x for x in entry.keys() if x != "name"]: new_entry[k] = entry[k] - self._data['instruction_forms'].append(new_entry) + self._data["instruction_forms"].append(new_entry) # remove old entry - self._data['instruction_forms'].remove(entry) + self._data["instruction_forms"].remove(entry) # Normalize instruction_form names (to UPPERCASE) and build dict for faster access: - self._data['instruction_forms_dict'] = defaultdict(list) - for iform in self._data['instruction_forms']: - iform['name'] = iform['name'].upper() - self._data['instruction_forms_dict'][iform['name']].append(iform) - self._data['internal_version'] = self.INTERNAL_VERSION + self._data["instruction_forms_dict"] = defaultdict(list) + for iform in self._data["instruction_forms"]: + iform["name"] = iform["name"].upper() + self._data["instruction_forms_dict"][iform["name"]].append(iform) + self._data["internal_version"] = self.INTERNAL_VERSION if not lazy: # cache internal representation for future use @@ -108,22 +109,24 @@ class MachineModel(object): # For use with dict instead of list as DB if name is None: return None - name_matched_iforms = self._data['instruction_forms_dict'].get(name.upper(), []) + name_matched_iforms = self._data["instruction_forms_dict"].get(name.upper(), []) try: return next( instruction_form - for instruction_form in name_matched_iforms if self._match_operands( - instruction_form['operands'] if 'operands' in instruction_form else [], - operands)) + for instruction_form in name_matched_iforms + if self._match_operands( + instruction_form["operands"] if "operands" in instruction_form else [], operands + ) + ) except StopIteration: return None except TypeError as e: - print('\nname: {}\noperands: {}'.format(name, operands)) + print("\nname: {}\noperands: {}".format(name, operands)) raise TypeError from e def average_port_pressure(self, port_pressure): """Construct average port pressure list from instruction data.""" - port_list = self._data['ports'] + port_list = self._data["ports"] average_pressure = [0.0] * len(port_list) for cycles, ports in port_pressure: for p in ports: @@ -138,59 +141,59 @@ class MachineModel(object): instr_data = self.get_instruction(name, operands) if instr_data is None: instr_data = {} - self._data['instruction_forms'].append(instr_data) + self._data["instruction_forms"].append(instr_data) - instr_data['name'] = name - instr_data['operands'] = operands - instr_data['latency'] = latency - instr_data['port_pressure'] = port_pressure - instr_data['throughput'] = throughput - instr_data['uops'] = uops + instr_data["name"] = name + instr_data["operands"] = operands + instr_data["latency"] = latency + instr_data["port_pressure"] = port_pressure + instr_data["throughput"] = throughput + instr_data["uops"] = uops def set_instruction_entry(self, entry): """Import instruction as entry object form information.""" self.set_instruction( - entry['name'], - entry['operands'] if 'operands' in entry else None, - entry['latency'] if 'latency' in entry else None, - entry['port_pressure'] if 'port_pressure' in entry else None, - entry['throughput'] if 'throughput' in entry else None, - entry['uops'] if 'uops' in entry else None, + entry["name"], + entry["operands"] if "operands" in entry else None, + entry["latency"] if "latency" in entry else None, + entry["port_pressure"] if "port_pressure" in entry else None, + entry["throughput"] if "throughput" in entry else None, + entry["uops"] if "uops" in entry else None, ) def add_port(self, port): """Add port in port model of current machine model.""" - if port not in self._data['ports']: - self._data['ports'].append(port) + if port not in self._data["ports"]: + self._data["ports"].append(port) def get_ISA(self): """Return ISA of :class:`MachineModel`.""" - return self._data['isa'].lower() + return self._data["isa"].lower() def get_arch(self): """Return micro-architecture code of :class:`MachineModel`.""" - return self._data['arch_code'].lower() + return self._data["arch_code"].lower() def get_ports(self): """Return port model of :class:`MachineModel`.""" - return self._data['ports'] + return self._data["ports"] def has_hidden_loads(self): """Return if model has hidden loads.""" - if 'hidden_loads' in self._data: - return self._data['hidden_loads'] + if "hidden_loads" in self._data: + return self._data["hidden_loads"] return False def get_load_latency(self, reg_type): """Return load latency for given register type.""" - return self._data['load_latency'][reg_type] if self._data['load_latency'][reg_type] else 0 + return self._data["load_latency"][reg_type] if self._data["load_latency"][reg_type] else 0 def get_load_throughput(self, memory): """Return load thorughput for given register type.""" - ld_tp = [m for m in self._data['load_throughput'] if self._match_mem_entries(memory, m)] + ld_tp = [m for m in self._data["load_throughput"] if self._match_mem_entries(memory, m)] if len(ld_tp) > 0: - return ld_tp[0]['port_pressure'].copy() - return self._data['load_throughput_default'].copy() + return ld_tp[0]["port_pressure"].copy() + return self._data["load_throughput_default"].copy() def get_store_latency(self, reg_type): """Return store latency for given register type.""" @@ -199,61 +202,61 @@ class MachineModel(object): def get_store_throughput(self, memory): """Return store throughput for given register type.""" - st_tp = [m for m in self._data['store_throughput'] if self._match_mem_entries(memory, m)] + st_tp = [m for m in self._data["store_throughput"] if self._match_mem_entries(memory, m)] if len(st_tp) > 0: - return st_tp[0]['port_pressure'].copy() - return self._data['store_throughput_default'].copy() + return st_tp[0]["port_pressure"].copy() + return self._data["store_throughput_default"].copy() def _match_mem_entries(self, mem, i_mem): """Check if memory addressing ``mem`` and ``i_mem`` are of the same type.""" - if self._data['isa'].lower() == 'aarch64': + if self._data["isa"].lower() == "aarch64": return self._is_AArch64_mem_type(i_mem, mem) - if self._data['isa'].lower() == 'x86': + if self._data["isa"].lower() == "x86": return self._is_x86_mem_type(i_mem, mem) def get_data_ports(self): """Return all data ports (i.e., ports with D-suffix) of current model.""" - data_port = re.compile(r'^[0-9]+D$') - data_ports = [x for x in filter(data_port.match, self._data['ports'])] + data_port = re.compile(r"^[0-9]+D$") + data_ports = [x for x in filter(data_port.match, self._data["ports"])] return data_ports @staticmethod def get_full_instruction_name(instruction_form): """Get one instruction name string including the mnemonic and all operands.""" operands = [] - for op in instruction_form['operands']: + for op in instruction_form["operands"]: op_attrs = [ - y + ':' + str(op[y]) - for y in list(filter(lambda x: True if x != 'class' else False, op)) + y + ":" + str(op[y]) + for y in list(filter(lambda x: True if x != "class" else False, op)) ] - operands.append('{}({})'.format(op['class'], ','.join(op_attrs))) - return '{} {}'.format(instruction_form['name'].lower(), ','.join(operands)) + operands.append("{}({})".format(op["class"], ",".join(op_attrs))) + return "{} {}".format(instruction_form["name"].lower(), ",".join(operands)) @staticmethod def get_isa_for_arch(arch): """Return ISA for given micro-arch ``arch``.""" arch_dict = { - 'a64fx': 'aarch64', - 'tx2': 'aarch64', - 'n1': 'aarch64', - 'zen1': 'x86', - 'zen+': 'x86', - 'zen2': 'x86', - 'con': 'x86', # Intel Conroe - 'wol': 'x86', # Intel Wolfdale - 'snb': 'x86', - 'ivb': 'x86', - 'hsw': 'x86', - 'bdw': 'x86', - 'skl': 'x86', - 'skx': 'x86', - 'csx': 'x86', - 'wsm': 'x86', - 'nhm': 'x86', - 'kbl': 'x86', - 'cnl': 'x86', - 'cfl': 'x86', - 'icl': 'x86', + "a64fx": "aarch64", + "tx2": "aarch64", + "n1": "aarch64", + "zen1": "x86", + "zen+": "x86", + "zen2": "x86", + "con": "x86", # Intel Conroe + "wol": "x86", # Intel Wolfdale + "snb": "x86", + "ivb": "x86", + "hsw": "x86", + "bdw": "x86", + "skl": "x86", + "skx": "x86", + "csx": "x86", + "wsm": "x86", + "nhm": "x86", + "kbl": "x86", + "cnl": "x86", + "cfl": "x86", + "icl": "x86", } arch = arch.lower() if arch in arch_dict: @@ -264,16 +267,16 @@ class MachineModel(object): def dump(self, stream=None): """Dump machine model to stream or return it as a ``str`` if no stream is given.""" # Replace instruction form's port_pressure with styled version for RoundtripDumper - formatted_instruction_forms = deepcopy(self._data['instruction_forms']) + formatted_instruction_forms = deepcopy(self._data["instruction_forms"]) for instruction_form in formatted_instruction_forms: - if instruction_form['port_pressure'] is not None: - cs = ruamel.yaml.comments.CommentedSeq(instruction_form['port_pressure']) + if instruction_form["port_pressure"] is not None: + cs = ruamel.yaml.comments.CommentedSeq(instruction_form["port_pressure"]) cs.fa.set_flow_style() - instruction_form['port_pressure'] = cs + instruction_form["port_pressure"] = cs # Replace load_throughput with styled version for RoundtripDumper formatted_load_throughput = [] - for lt in self._data['load_throughput']: + for lt in self._data["load_throughput"]: cm = ruamel.yaml.comments.CommentedMap(lt) cm.fa.set_flow_style() formatted_load_throughput.append(cm) @@ -287,13 +290,18 @@ class MachineModel(object): { k: v for k, v in self._data.items() - if k not in ['instruction_forms', 'instruction_forms_dict', 'load_throughput', - 'internal_version'] + if k + not in [ + "instruction_forms", + "instruction_forms_dict", + "load_throughput", + "internal_version", + ] }, stream, ) - yaml.dump({'load_throughput': formatted_load_throughput}, stream) - yaml.dump({'instruction_forms': formatted_instruction_forms}, stream) + yaml.dump({"load_throughput": formatted_load_throughput}, stream) + yaml.dump({"instruction_forms": formatted_instruction_forms}, stream) if isinstance(stream, StringIO): return stream.getvalue() @@ -312,21 +320,21 @@ class MachineModel(object): hexhash = hashlib.sha256(p.read_bytes()).hexdigest() # 1. companion cachefile: same location, with '._.pickle' - companion_cachefile = p.with_name('.' + p.stem + '_' + hexhash).with_suffix('.pickle') + companion_cachefile = p.with_name("." + p.stem + "_" + hexhash).with_suffix(".pickle") if companion_cachefile.exists(): # companion file (must be up-to-date, due to equal hash) - with companion_cachefile.open('rb') as f: + with companion_cachefile.open("rb") as f: data = pickle.load(f) - if data.get('internal_version') == self.INTERNAL_VERSION: + if data.get("internal_version") == self.INTERNAL_VERSION: return data # 2. home cachefile: ~/.osaca/cache/_.pickle - home_cachefile = (Path(utils.CACHE_DIR) / (p.stem + '_' + hexhash)).with_suffix('.pickle') + home_cachefile = (Path(utils.CACHE_DIR) / (p.stem + "_" + hexhash)).with_suffix(".pickle") if home_cachefile.exists(): # home file (must be up-to-date, due to equal hash) - with home_cachefile.open('rb') as f: + with home_cachefile.open("rb") as f: data = pickle.load(f) - if data.get('internal_version') == self.INTERNAL_VERSION: + if data.get("internal_version") == self.INTERNAL_VERSION: return data return False @@ -340,9 +348,9 @@ class MachineModel(object): p = Path(filepath) hexhash = hashlib.sha256(p.read_bytes()).hexdigest() # 1. companion cachefile: same location, with '._.pickle' - companion_cachefile = p.with_name('.' + p.stem + '_' + hexhash).with_suffix('.pickle') + companion_cachefile = p.with_name("." + p.stem + "_" + hexhash).with_suffix(".pickle") if os.access(str(companion_cachefile.parent), os.W_OK): - with companion_cachefile.open('wb') as f: + with companion_cachefile.open("wb") as f: pickle.dump(self._data, f) return @@ -352,92 +360,92 @@ class MachineModel(object): os.makedirs(cache_dir, exist_ok=True) except OSError: return - home_cachefile = (cache_dir / (p.stem + '_' + hexhash)).with_suffix('.pickle') + home_cachefile = (cache_dir / (p.stem + "_" + hexhash)).with_suffix(".pickle") if os.access(str(home_cachefile.parent), os.W_OK): - with home_cachefile.open('wb') as f: + with home_cachefile.open("wb") as f: pickle.dump(self._data, f) def _get_key(self, name, operands): """Get unique instruction form key for dict DB.""" - key_string = name.lower() + '-' + key_string = name.lower() + "-" if operands is None: return key_string[:-1] - key_string += '_'.join([self._get_operand_hash(op) for op in operands]) + key_string += "_".join([self._get_operand_hash(op) for op in operands]) return key_string def _get_operand_hash(self, operand): """Get unique key for operand for dict DB""" - operand_string = '' - if 'class' in operand: + operand_string = "" + if "class" in operand: # DB entry - opclass = operand['class'] + opclass = operand["class"] else: # parsed instruction opclass = list(operand.keys())[0] operand = operand[opclass] - if opclass == 'immediate': + if opclass == "immediate": # Immediate - operand_string += 'i' - elif opclass == 'register': + operand_string += "i" + elif opclass == "register": # Register - if 'prefix' in operand: - operand_string += operand['prefix'] - operand_string += operand['shape'] if 'shape' in operand else '' - elif 'name' in operand: - operand_string += 'r' if operand['name'] == 'gpr' else operand['name'][0] - elif opclass == 'memory': + if "prefix" in operand: + operand_string += operand["prefix"] + operand_string += operand["shape"] if "shape" in operand else "" + elif "name" in operand: + operand_string += "r" if operand["name"] == "gpr" else operand["name"][0] + elif opclass == "memory": # Memory - operand_string += 'm' - operand_string += 'b' if operand['base'] is not None else '' - operand_string += 'o' if operand['offset'] is not None else '' - operand_string += 'i' if operand['index'] is not None else '' + operand_string += "m" + operand_string += "b" if operand["base"] is not None else "" + operand_string += "o" if operand["offset"] is not None else "" + operand_string += "i" if operand["index"] is not None else "" operand_string += ( - 's' if operand['scale'] == self.WILDCARD or operand['scale'] > 1 else '' + "s" if operand["scale"] == self.WILDCARD or operand["scale"] > 1 else "" ) - if 'pre-indexed' in operand: - operand_string += 'r' if operand['pre-indexed'] else '' - operand_string += 'p' if operand['post-indexed'] else '' + if "pre-indexed" in operand: + operand_string += "r" if operand["pre-indexed"] else "" + operand_string += "p" if operand["post-indexed"] else "" return operand_string def _create_db_operand_aarch64(self, operand): """Create instruction form operand for DB out of operand string.""" - if operand == 'i': - return {'class': 'immediate', 'imd': 'int'} - elif operand in 'wxbhsdq': - return {'class': 'register', 'prefix': operand} - elif operand.startswith('v'): - return {'class': 'register', 'prefix': 'v', 'shape': operand[1:2]} - elif operand.startswith('m'): + if operand == "i": + return {"class": "immediate", "imd": "int"} + elif operand in "wxbhsdq": + return {"class": "register", "prefix": operand} + elif operand.startswith("v"): + return {"class": "register", "prefix": "v", "shape": operand[1:2]} + elif operand.startswith("m"): return { - 'class': 'memory', - 'base': 'x' if 'b' in operand else None, - 'offset': 'imd' if 'o' in operand else None, - 'index': 'gpr' if 'i' in operand else None, - 'scale': 8 if 's' in operand else 1, - 'pre-indexed': True if 'r' in operand else False, - 'post-indexed': True if 'p' in operand else False, + "class": "memory", + "base": "x" if "b" in operand else None, + "offset": "imd" if "o" in operand else None, + "index": "gpr" if "i" in operand else None, + "scale": 8 if "s" in operand else 1, + "pre-indexed": True if "r" in operand else False, + "post-indexed": True if "p" in operand else False, } else: - raise ValueError('Parameter {} is not a valid operand code'.format(operand)) + raise ValueError("Parameter {} is not a valid operand code".format(operand)) def _create_db_operand_x86(self, operand): """Create instruction form operand for DB out of operand string.""" - if operand == 'r': - return {'class': 'register', 'name': 'gpr'} - elif operand in 'xyz': - return {'class': 'register', 'name': operand + 'mm'} - elif operand == 'i': - return {'class': 'immediate', 'imd': 'int'} - elif operand.startswith('m'): + if operand == "r": + return {"class": "register", "name": "gpr"} + elif operand in "xyz": + return {"class": "register", "name": operand + "mm"} + elif operand == "i": + return {"class": "immediate", "imd": "int"} + elif operand.startswith("m"): return { - 'class': 'memory', - 'base': 'gpr' if 'b' in operand else None, - 'offset': 'imd' if 'o' in operand else None, - 'index': 'gpr' if 'i' in operand else None, - 'scale': 8 if 's' in operand else 1, + "class": "memory", + "base": "gpr" if "b" in operand else None, + "offset": "imd" if "o" in operand else None, + "index": "gpr" if "i" in operand else None, + "scale": 8 if "s" in operand else 1, } else: - raise ValueError('Parameter {} is not a valid operand code'.format(operand)) + raise ValueError("Parameter {} is not a valid operand code".format(operand)) def _check_for_duplicate(self, name, operands): """ @@ -450,9 +458,9 @@ class MachineModel(object): """ matches = [ instruction_form - for instruction_form in self._data['instruction_forms'] - if instruction_form['name'].lower() == name.lower() - and self._match_operands(instruction_form['operands'], operands) + for instruction_form in self._data["instruction_forms"] + if instruction_form["name"].lower() == name.lower() + and self._match_operands(instruction_form["operands"], operands) ] if len(matches) > 1: return True @@ -475,80 +483,76 @@ class MachineModel(object): """Check if the types of operand ``i_operand`` and ``operand`` match.""" # check for wildcard if self.WILDCARD in operand: - if ( - 'class' in i_operand - and i_operand['class'] == 'register' - or 'register' in i_operand - ): + if "class" in i_operand and i_operand["class"] == "register" or "register" in i_operand: return True else: return False - if self._data['isa'].lower() == 'aarch64': + if self._data["isa"].lower() == "aarch64": return self._check_AArch64_operands(i_operand, operand) - if self._data['isa'].lower() == 'x86': + if self._data["isa"].lower() == "x86": return self._check_x86_operands(i_operand, operand) def _check_AArch64_operands(self, i_operand, operand): """Check if the types of operand ``i_operand`` and ``operand`` match.""" - if 'class' in operand: + if "class" in operand: # compare two DB entries return self._compare_db_entries(i_operand, operand) # TODO support class wildcards # register - if 'register' in operand: - if i_operand['class'] != 'register': + if "register" in operand: + if i_operand["class"] != "register": return False - return self._is_AArch64_reg_type(i_operand, operand['register']) + return self._is_AArch64_reg_type(i_operand, operand["register"]) # memory - if 'memory' in operand: - if i_operand['class'] != 'memory': + if "memory" in operand: + if i_operand["class"] != "memory": return False - return self._is_AArch64_mem_type(i_operand, operand['memory']) + return self._is_AArch64_mem_type(i_operand, operand["memory"]) # immediate # TODO support wildcards - if 'value' in operand or ('immediate' in operand and 'value' in operand['immediate']): - return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int' - if 'float' in operand or ('immediate' in operand and 'float' in operand['immediate']): - return i_operand['class'] == 'immediate' and i_operand['imd'] == 'float' - if 'double' in operand or ('immediate' in operand and 'double' in operand['immediate']): - return i_operand['class'] == 'immediate' and i_operand['imd'] == 'double' + if "value" in operand or ("immediate" in operand and "value" in operand["immediate"]): + return i_operand["class"] == "immediate" and i_operand["imd"] == "int" + if "float" in operand or ("immediate" in operand and "float" in operand["immediate"]): + return i_operand["class"] == "immediate" and i_operand["imd"] == "float" + if "double" in operand or ("immediate" in operand and "double" in operand["immediate"]): + return i_operand["class"] == "immediate" and i_operand["imd"] == "double" # identifier - if 'identifier' in operand or ( - 'immediate' in operand and 'identifier' in operand['immediate'] + if "identifier" in operand or ( + "immediate" in operand and "identifier" in operand["immediate"] ): - return i_operand['class'] == 'identifier' + return i_operand["class"] == "identifier" # prefetch option - if 'prfop' in operand: - return i_operand['class'] == 'prfop' + if "prfop" in operand: + return i_operand["class"] == "prfop" # no match return False def _check_x86_operands(self, i_operand, operand): """Check if the types of operand ``i_operand`` and ``operand`` match.""" - if 'class' in operand: + if "class" in operand: # compare two DB entries return self._compare_db_entries(i_operand, operand) # register - if 'register' in operand: - if i_operand['class'] != 'register': + if "register" in operand: + if i_operand["class"] != "register": return False - return self._is_x86_reg_type(i_operand, operand['register'], consider_masking=True) + return self._is_x86_reg_type(i_operand, operand["register"], consider_masking=True) # memory - if 'memory' in operand: - if i_operand['class'] != 'memory': + if "memory" in operand: + if i_operand["class"] != "memory": return False - return self._is_x86_mem_type(i_operand, operand['memory']) + return self._is_x86_mem_type(i_operand, operand["memory"]) # immediate - if 'immediate' in operand or 'value' in operand: - return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int' + if "immediate" in operand or "value" in operand: + return i_operand["class"] == "immediate" and i_operand["imd"] == "int" # identifier (e.g., labels) - if 'identifier' in operand: - return i_operand['class'] == 'identifier' + if "identifier" in operand: + return i_operand["class"] == "identifier" def _compare_db_entries(self, operand_1, operand_2): """Check if operand types in DB format (i.e., not parsed) match.""" operand_attributes = list( - filter(lambda x: True if x != 'source' and x != 'destination' else False, operand_1) + filter(lambda x: True if x != "source" and x != "destination" else False, operand_1) ) for key in operand_attributes: try: @@ -563,22 +567,21 @@ class MachineModel(object): def _is_AArch64_reg_type(self, i_reg, reg): """Check if register type match.""" # check for wildcards - if reg['prefix'] == self.WILDCARD or i_reg['prefix'] == self.WILDCARD: - if 'shape' in reg: - if 'shape' in i_reg and ( - reg['shape'] == i_reg['shape'] - or self.WILDCARD in (reg['shape'] + i_reg['shape']) + if reg["prefix"] == self.WILDCARD or i_reg["prefix"] == self.WILDCARD: + if "shape" in reg: + if "shape" in i_reg and ( + reg["shape"] == i_reg["shape"] + or self.WILDCARD in (reg["shape"] + i_reg["shape"]) ): return True return False return True # check for prefix and shape - if reg['prefix'] != i_reg['prefix']: + if reg["prefix"] != i_reg["prefix"]: return False - if 'shape' in reg: - if 'shape' in i_reg and ( - reg['shape'] == i_reg['shape'] - or self.WILDCARD in (reg['shape'] + i_reg['shape']) + if "shape" in reg: + if "shape" in i_reg and ( + reg["shape"] == i_reg["shape"] or self.WILDCARD in (reg["shape"] + i_reg["shape"]) ): return True return False @@ -586,48 +589,48 @@ class MachineModel(object): def _is_x86_reg_type(self, i_reg, reg, consider_masking=False): """Check if register type match.""" - i_reg_name = i_reg['name'] if i_reg and 'name' in i_reg else i_reg + i_reg_name = i_reg["name"] if i_reg and "name" in i_reg else i_reg if reg is None: if i_reg is None: return True return False # check for wildcards - if i_reg_name == self.WILDCARD or reg['name'] == self.WILDCARD: + if i_reg_name == self.WILDCARD or reg["name"] == self.WILDCARD: return True # differentiate between vector registers (mm, xmm, ymm, zmm) and others (gpr) parser_x86 = ParserX86ATT() if parser_x86.is_vector_register(reg): - if reg['name'].rstrip(string.digits).lower() == i_reg_name: + if reg["name"].rstrip(string.digits).lower() == i_reg_name: # Consider masking and zeroing for AVX512 if consider_masking: mask_ok = zero_ok = True - if 'mask' in reg or 'mask' in i_reg: + if "mask" in reg or "mask" in i_reg: # one instruction is missing the masking while the other has it mask_ok = False # check for wildcard if ( ( - 'mask' in reg - and reg['mask'].rstrip(string.digits).lower() == i_reg.get('mask') + "mask" in reg + and reg["mask"].rstrip(string.digits).lower() == i_reg.get("mask") ) - or reg.get('mask') == self.WILDCARD - or i_reg.get('mask') == self.WILDCARD + or reg.get("mask") == self.WILDCARD + or i_reg.get("mask") == self.WILDCARD ): mask_ok = True - if bool('zeroing' in reg) ^ bool('zeroing' in i_reg): + if bool("zeroing" in reg) ^ bool("zeroing" in i_reg): # one instruction is missing zeroing while the other has it zero_ok = False # check for wildcard if ( - i_reg.get('zeroing') == self.WILDCARD - or reg.get('zeroing') == self.WILDCARD + i_reg.get("zeroing") == self.WILDCARD + or reg.get("zeroing") == self.WILDCARD ): zero_ok = True if not mask_ok or not zero_ok: return False return True else: - if i_reg_name == 'gpr': + if i_reg_name == "gpr": return True return False @@ -636,50 +639,50 @@ class MachineModel(object): if ( # check base ( - (mem['base'] is None and i_mem['base'] is None) - or i_mem['base'] == self.WILDCARD - or mem['base']['prefix'] == i_mem['base'] + (mem["base"] is None and i_mem["base"] is None) + or i_mem["base"] == self.WILDCARD + or mem["base"]["prefix"] == i_mem["base"] ) # check offset and ( - mem['offset'] == i_mem['offset'] - or i_mem['offset'] == self.WILDCARD + mem["offset"] == i_mem["offset"] + or i_mem["offset"] == self.WILDCARD or ( - mem['offset'] is not None - and 'identifier' in mem['offset'] - and i_mem['offset'] == 'identifier' + mem["offset"] is not None + and "identifier" in mem["offset"] + and i_mem["offset"] == "identifier" ) or ( - mem['offset'] is not None - and 'value' in mem['offset'] - and i_mem['offset'] == 'imd' + mem["offset"] is not None + and "value" in mem["offset"] + and i_mem["offset"] == "imd" ) ) # check index and ( - mem['index'] == i_mem['index'] - or i_mem['index'] == self.WILDCARD + mem["index"] == i_mem["index"] + or i_mem["index"] == self.WILDCARD or ( - mem['index'] is not None - and 'prefix' in mem['index'] - and mem['index']['prefix'] == i_mem['index'] + mem["index"] is not None + and "prefix" in mem["index"] + and mem["index"]["prefix"] == i_mem["index"] ) ) # check scale and ( - mem['scale'] == i_mem['scale'] - or i_mem['scale'] == self.WILDCARD - or (mem['scale'] != 1 and i_mem['scale'] != 1) + mem["scale"] == i_mem["scale"] + or i_mem["scale"] == self.WILDCARD + or (mem["scale"] != 1 and i_mem["scale"] != 1) ) # check pre-indexing and ( - i_mem['pre-indexed'] == self.WILDCARD - or ('pre_indexed' in mem) == (i_mem['pre-indexed']) + i_mem["pre-indexed"] == self.WILDCARD + or ("pre_indexed" in mem) == (i_mem["pre-indexed"]) ) # check post-indexing and ( - i_mem['post-indexed'] == self.WILDCARD - or ('post_indexed' in mem) == (i_mem['post-indexed']) + i_mem["post-indexed"] == self.WILDCARD + or ("post_indexed" in mem) == (i_mem["post-indexed"]) ) ): return True @@ -690,48 +693,48 @@ class MachineModel(object): if ( # check base ( - (mem['base'] is None and i_mem['base'] is None) - or i_mem['base'] == self.WILDCARD - or self._is_x86_reg_type(i_mem['base'], mem['base']) + (mem["base"] is None and i_mem["base"] is None) + or i_mem["base"] == self.WILDCARD + or self._is_x86_reg_type(i_mem["base"], mem["base"]) ) # check offset and ( - mem['offset'] == i_mem['offset'] - or i_mem['offset'] == self.WILDCARD + mem["offset"] == i_mem["offset"] + or i_mem["offset"] == self.WILDCARD or ( - mem['offset'] is not None - and 'identifier' in mem['offset'] - and i_mem['offset'] == 'identifier' + mem["offset"] is not None + and "identifier" in mem["offset"] + and i_mem["offset"] == "identifier" ) or ( - mem['offset'] is not None - and 'value' in mem['offset'] + mem["offset"] is not None + and "value" in mem["offset"] and ( - i_mem['offset'] == 'imd' - or (i_mem['offset'] is None and mem['offset']['value'] == '0') + i_mem["offset"] == "imd" + or (i_mem["offset"] is None and mem["offset"]["value"] == "0") ) ) or ( - mem['offset'] is not None - and 'identifier' in mem['offset'] - and i_mem['offset'] == 'id' + mem["offset"] is not None + and "identifier" in mem["offset"] + and i_mem["offset"] == "id" ) ) # check index and ( - mem['index'] == i_mem['index'] - or i_mem['index'] == self.WILDCARD + mem["index"] == i_mem["index"] + or i_mem["index"] == self.WILDCARD or ( - mem['index'] is not None - and 'name' in mem['index'] - and self._is_x86_reg_type(i_mem['index'], mem['index']) + mem["index"] is not None + and "name" in mem["index"] + and self._is_x86_reg_type(i_mem["index"], mem["index"]) ) ) # check scale and ( - mem['scale'] == i_mem['scale'] - or i_mem['scale'] == self.WILDCARD - or (mem['scale'] != 1 and i_mem['scale'] != 1) + mem["scale"] == i_mem["scale"] + or i_mem["scale"] == self.WILDCARD + or (mem["scale"] != 1 and i_mem["scale"] != 1) ) ): return True @@ -748,4 +751,4 @@ class MachineModel(object): def __represent_none(self, yaml_obj, data): """YAML representation for `None`""" - return yaml_obj.represent_scalar(u'tag:yaml.org,2002:null', u'~') + return yaml_obj.represent_scalar(u"tag:yaml.org,2002:null", u"~") diff --git a/osaca/semantics/isa_semantics.py b/osaca/semantics/isa_semantics.py index f475009..de3e5d2 100755 --- a/osaca/semantics/isa_semantics.py +++ b/osaca/semantics/isa_semantics.py @@ -12,25 +12,25 @@ class INSTR_FLAGS: Flags used for unknown or special instructions """ - LD = 'is_load_instruction' - TP_UNKWN = 'tp_unknown' - LT_UNKWN = 'lt_unknown' - NOT_BOUND = 'not_bound' - HIDDEN_LD = 'hidden_load' - HAS_LD = 'performs_load' - HAS_ST = 'performs_store' + LD = "is_load_instruction" + TP_UNKWN = "tp_unknown" + LT_UNKWN = "lt_unknown" + NOT_BOUND = "not_bound" + HIDDEN_LD = "hidden_load" + HAS_LD = "performs_load" + HAS_ST = "performs_store" class ISASemantics(object): - GAS_SUFFIXES = 'bswlqt' + GAS_SUFFIXES = "bswlqt" def __init__(self, isa, path_to_yaml=None): self._isa = isa.lower() - path = path_to_yaml or utils.find_datafile('isa/' + self._isa + '.yml') + path = path_to_yaml or utils.find_datafile("isa/" + self._isa + ".yml") self._isa_model = MachineModel(path_to_yaml=path) - if self._isa == 'x86': + if self._isa == "x86": self._parser = ParserX86ATT() - elif self._isa == 'aarch64': + elif self._isa == "aarch64": self._parser = ParserAArch64() def process(self, instruction_forms): @@ -45,26 +45,26 @@ class ISASemantics(object): def assign_src_dst(self, instruction_form): """Update instruction form dictionary with source, destination and flag information.""" # if the instruction form doesn't have operands or is None, there's nothing to do - if instruction_form['operands'] is None or instruction_form['instruction'] is None: - instruction_form['semantic_operands'] = AttrDict( - {'source': [], 'destination': [], 'src_dst': []} + if instruction_form["operands"] is None or instruction_form["instruction"] is None: + instruction_form["semantic_operands"] = AttrDict( + {"source": [], "destination": [], "src_dst": []} ) return # check if instruction form is in ISA yaml, otherwise apply standard operand assignment # (one dest, others source) isa_data = self._isa_model.get_instruction( - instruction_form['instruction'], instruction_form['operands'] + instruction_form["instruction"], instruction_form["operands"] ) if ( isa_data is None - and self._isa == 'x86' - and instruction_form['instruction'][-1] in self.GAS_SUFFIXES + and self._isa == "x86" + and instruction_form["instruction"][-1] in self.GAS_SUFFIXES ): # Check for instruction without GAS suffix isa_data = self._isa_model.get_instruction( - instruction_form['instruction'][:-1], instruction_form['operands'] + instruction_form["instruction"][:-1], instruction_form["operands"] ) - operands = instruction_form['operands'] + operands = instruction_form["operands"] op_dict = {} assign_default = False if isa_data: @@ -74,52 +74,52 @@ class ISASemantics(object): # Couldn't found instruction form in ISA DB assign_default = True # check for equivalent register-operands DB entry if LD/ST - if any(['memory' in op for op in operands]): - operands_reg = self.substitute_mem_address(instruction_form['operands']) + if any(["memory" in op for op in operands]): + operands_reg = self.substitute_mem_address(instruction_form["operands"]) isa_data_reg = self._isa_model.get_instruction( - instruction_form['instruction'], operands_reg + instruction_form["instruction"], operands_reg ) if ( isa_data_reg is None - and self._isa == 'x86' - and instruction_form['instruction'][-1] in self.GAS_SUFFIXES + and self._isa == "x86" + and instruction_form["instruction"][-1] in self.GAS_SUFFIXES ): # Check for instruction without GAS suffix isa_data_reg = self._isa_model.get_instruction( - instruction_form['instruction'][:-1], operands_reg + instruction_form["instruction"][:-1], operands_reg ) if isa_data_reg: assign_default = False op_dict = self._apply_found_ISA_data(isa_data_reg, operands) if assign_default: # no irregular operand structure, apply default - op_dict['source'] = self._get_regular_source_operands(instruction_form) - op_dict['destination'] = self._get_regular_destination_operands(instruction_form) - op_dict['src_dst'] = [] + op_dict["source"] = self._get_regular_source_operands(instruction_form) + op_dict["destination"] = self._get_regular_destination_operands(instruction_form) + op_dict["src_dst"] = [] # post-process pre- and post-indexing for aarch64 memory operands - if self._isa == 'aarch64': - for operand in [op for op in op_dict['source'] if 'memory' in op]: - if ('post_indexed' in operand['memory'] and operand['memory']['post_indexed']) or ( - 'pre_indexed' in operand['memory'] and operand['memory']['pre_indexed'] + if self._isa == "aarch64": + for operand in [op for op in op_dict["source"] if "memory" in op]: + if ("post_indexed" in operand["memory"] and operand["memory"]["post_indexed"]) or ( + "pre_indexed" in operand["memory"] and operand["memory"]["pre_indexed"] ): - op_dict['src_dst'].append(AttrDict.convert_dict( - {'register': operand['memory']['base']})) - for operand in [op for op in op_dict['destination'] if 'memory' in op]: - if ('post_indexed' in operand['memory'] and operand['memory']['post_indexed']) or ( - 'pre_indexed' in operand['memory'] and operand['memory']['pre_indexed'] + op_dict["src_dst"].append( + AttrDict.convert_dict({"register": operand["memory"]["base"]}) + ) + for operand in [op for op in op_dict["destination"] if "memory" in op]: + if ("post_indexed" in operand["memory"] and operand["memory"]["post_indexed"]) or ( + "pre_indexed" in operand["memory"] and operand["memory"]["pre_indexed"] ): - op_dict['src_dst'].append(AttrDict.convert_dict( - {'register': operand['memory']['base']})) + op_dict["src_dst"].append( + AttrDict.convert_dict({"register": operand["memory"]["base"]}) + ) # store operand list in dict and reassign operand key/value pair - instruction_form['semantic_operands'] = AttrDict.convert_dict(op_dict) + instruction_form["semantic_operands"] = AttrDict.convert_dict(op_dict) # assign LD/ST flags - instruction_form['flags'] = ( - instruction_form['flags'] if 'flags' in instruction_form else [] - ) + instruction_form["flags"] = instruction_form["flags"] if "flags" in instruction_form else [] if self._has_load(instruction_form): - instruction_form['flags'] += [INSTR_FLAGS.HAS_LD] + instruction_form["flags"] += [INSTR_FLAGS.HAS_LD] if self._has_store(instruction_form): - instruction_form['flags'] += [INSTR_FLAGS.HAS_ST] + instruction_form["flags"] += [INSTR_FLAGS.HAS_ST] def _apply_found_ISA_data(self, isa_data, operands): """ @@ -131,34 +131,34 @@ class ISASemantics(object): :returns: `dict` -- operands dictionary with src/dst assignment """ op_dict = {} - op_dict['source'] = [] - op_dict['destination'] = [] - op_dict['src_dst'] = [] - for i, op in enumerate(isa_data['operands']): - if op['source'] and op['destination']: - op_dict['src_dst'].append(operands[i]) + op_dict["source"] = [] + op_dict["destination"] = [] + op_dict["src_dst"] = [] + for i, op in enumerate(isa_data["operands"]): + if op["source"] and op["destination"]: + op_dict["src_dst"].append(operands[i]) continue - if op['source']: - op_dict['source'].append(operands[i]) + if op["source"]: + op_dict["source"].append(operands[i]) continue - if op['destination']: - op_dict['destination'].append(operands[i]) + if op["destination"]: + op_dict["destination"].append(operands[i]) continue # check for hidden operands like flags or registers - if 'hidden_operands' in isa_data: + if "hidden_operands" in isa_data: # add operand(s) to semantic_operands of instruction form - for op in isa_data['hidden_operands']: + for op in isa_data["hidden_operands"]: dict_key = ( - 'src_dst' - if op['source'] and op['destination'] - else 'source' - if op['source'] - else 'destination' + "src_dst" + if op["source"] and op["destination"] + else "source" + if op["source"] + else "destination" ) - hidden_op = {op['class']: {}} - key_filter = ['class', 'source', 'destination'] + hidden_op = {op["class"]: {}} + key_filter = ["class", "source", "destination"] for key in [k for k in op.keys() if k not in key_filter]: - hidden_op[op['class']][key] = op[key] + hidden_op[op["class"]][key] = op[key] hidden_op = AttrDict.convert_dict(hidden_op) op_dict[dict_key].append(hidden_op) return op_dict @@ -166,54 +166,54 @@ class ISASemantics(object): def _has_load(self, instruction_form): """Check if instruction form performs a LOAD""" for operand in chain( - instruction_form['semantic_operands']['source'], - instruction_form['semantic_operands']['src_dst'], + instruction_form["semantic_operands"]["source"], + instruction_form["semantic_operands"]["src_dst"], ): - if 'memory' in operand: + if "memory" in operand: return True return False def _has_store(self, instruction_form): """Check if instruction form perfroms a STORE""" for operand in chain( - instruction_form['semantic_operands']['destination'], - instruction_form['semantic_operands']['src_dst'], + instruction_form["semantic_operands"]["destination"], + instruction_form["semantic_operands"]["src_dst"], ): - if 'memory' in operand: + if "memory" in operand: return True return False def _get_regular_source_operands(self, instruction_form): """Get source operand of given instruction form assuming regular src/dst behavior.""" # if there is only one operand, assume it is a source operand - if len(instruction_form['operands']) == 1: - return [instruction_form['operands'][0]] - if self._isa == 'x86': + if len(instruction_form["operands"]) == 1: + return [instruction_form["operands"][0]] + if self._isa == "x86": # return all but last operand - return [op for op in instruction_form['operands'][0:-1]] - elif self._isa == 'aarch64': - return [op for op in instruction_form['operands'][1:]] + return [op for op in instruction_form["operands"][0:-1]] + elif self._isa == "aarch64": + return [op for op in instruction_form["operands"][1:]] else: raise ValueError("Unsupported ISA {}.".format(self._isa)) def _get_regular_destination_operands(self, instruction_form): """Get destination operand of given instruction form assuming regular src/dst behavior.""" # if there is only one operand, assume no destination - if len(instruction_form['operands']) == 1: + if len(instruction_form["operands"]) == 1: return [] - if self._isa == 'x86': + if self._isa == "x86": # return last operand - return instruction_form['operands'][-1:] - if self._isa == 'aarch64': + return instruction_form["operands"][-1:] + if self._isa == "aarch64": # return first operand - return instruction_form['operands'][:1] + return instruction_form["operands"][:1] else: raise ValueError("Unsupported ISA {}.".format(self._isa)) def substitute_mem_address(self, operands): """Create memory wildcard for all memory operands""" - return [self._create_reg_wildcard() if 'memory' in op else op for op in operands] + return [self._create_reg_wildcard() if "memory" in op else op for op in operands] def _create_reg_wildcard(self): """Wildcard constructor""" - return {'*': '*'} + return {"*": "*"} diff --git a/osaca/semantics/kernel_dg.py b/osaca/semantics/kernel_dg.py index 905f9f3..7505a46 100755 --- a/osaca/semantics/kernel_dg.py +++ b/osaca/semantics/kernel_dg.py @@ -30,34 +30,34 @@ class KernelDG(nx.DiGraph): # 3. get LT value and set as edge weight dg = nx.DiGraph() for i, instruction_form in enumerate(kernel): - dg.add_node(instruction_form['line_number']) - dg.nodes[instruction_form['line_number']]['instruction_form'] = instruction_form + dg.add_node(instruction_form["line_number"]) + dg.nodes[instruction_form["line_number"]]["instruction_form"] = instruction_form # add load as separate node if existent if ( - INSTR_FLAGS.HAS_LD in instruction_form['flags'] - and INSTR_FLAGS.LD not in instruction_form['flags'] + INSTR_FLAGS.HAS_LD in instruction_form["flags"] + and INSTR_FLAGS.LD not in instruction_form["flags"] ): # add new node - dg.add_node(instruction_form['line_number'] + 0.1) - dg.nodes[instruction_form['line_number'] + 0.1][ - 'instruction_form' + dg.add_node(instruction_form["line_number"] + 0.1) + dg.nodes[instruction_form["line_number"] + 0.1][ + "instruction_form" ] = instruction_form # and set LD latency as edge weight dg.add_edge( - instruction_form['line_number'] + 0.1, - instruction_form['line_number'], - latency=instruction_form['latency'] - instruction_form['latency_wo_load'], + instruction_form["line_number"] + 0.1, + instruction_form["line_number"], + latency=instruction_form["latency"] - instruction_form["latency_wo_load"], ) for dep in self.find_depending(instruction_form, kernel[i + 1 :]): edge_weight = ( - instruction_form['latency'] - if 'latency_wo_load' not in instruction_form - else instruction_form['latency_wo_load'] + instruction_form["latency"] + if "latency_wo_load" not in instruction_form + else instruction_form["latency_wo_load"] ) dg.add_edge( - instruction_form['line_number'], dep['line_number'], latency=edge_weight + instruction_form["line_number"], dep["line_number"], latency=edge_weight ) - dg.nodes[dep['line_number']]['instruction_form'] = dep + dg.nodes[dep["line_number"]]["instruction_form"] = dep return dg def check_for_loopcarried_dep(self, kernel): @@ -109,17 +109,17 @@ class KernelDG(nx.DiGraph): for dep in loopcarried_deps: nodes = [] for n in dep[1]: - self._get_node_by_lineno(int(n))['latency_lcd'] = 0 + self._get_node_by_lineno(int(n))["latency_lcd"] = 0 for n in dep[1]: node = self._get_node_by_lineno(int(n)) if int(n) != n and int(n) in dep[1]: - node['latency_lcd'] += node['latency'] - node['latency_wo_load'] + node["latency_lcd"] += node["latency"] - node["latency_wo_load"] else: - node['latency_lcd'] += node['latency_wo_load'] + node["latency_lcd"] += node["latency_wo_load"] nodes.append(node) loopcarried_deps_dict[dep[0]] = { - 'root': self._get_node_by_lineno(dep[0]), - 'dependencies': nodes, + "root": self._get_node_by_lineno(dep[0]), + "dependencies": nodes, } return loopcarried_deps_dict @@ -131,30 +131,30 @@ class KernelDG(nx.DiGraph): def get_critical_path(self): """Find and return critical path after the creation of a directed graph.""" if nx.algorithms.dag.is_directed_acyclic_graph(self.dg): - longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight='latency') + longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight="latency") for line_number in longest_path: - self._get_node_by_lineno(int(line_number))['latency_cp'] = 0 + self._get_node_by_lineno(int(line_number))["latency_cp"] = 0 # add LD latency to instruction for line_number in longest_path: node = self._get_node_by_lineno(int(line_number)) if line_number != int(line_number) and int(line_number) in longest_path: - node['latency_cp'] += self.dg.edges[(line_number, int(line_number))]['latency'] + node["latency_cp"] += self.dg.edges[(line_number, int(line_number))]["latency"] elif ( line_number == int(line_number) - and 'mem_dep' in node - and self.dg.has_edge(node['mem_dep']['line_number'], line_number) + and "mem_dep" in node + and self.dg.has_edge(node["mem_dep"]["line_number"], line_number) ): - node['latency_cp'] += node['latency'] + node["latency_cp"] += node["latency"] else: - node['latency_cp'] += ( - node['latency'] - if 'latency_wo_load' not in node - else node['latency_wo_load'] + node["latency_cp"] += ( + node["latency"] + if "latency_wo_load" not in node + else node["latency_wo_load"] ) - return [x for x in self.kernel if x['line_number'] in longest_path] + return [x for x in self.kernel if x["line_number"] in longest_path] else: # split to DAG - raise NotImplementedError('Kernel is cyclic.') + raise NotImplementedError("Kernel is cyclic.") def get_loopcarried_dependencies(self): """ @@ -164,7 +164,7 @@ class KernelDG(nx.DiGraph): return self.loopcarried_deps else: # split to DAG - raise NotImplementedError('Kernel is cyclic.') + raise NotImplementedError("Kernel is cyclic.") def find_depending( self, instruction_form, kernel, include_write=False, flag_dependencies=False @@ -186,7 +186,7 @@ class KernelDG(nx.DiGraph): instruction_form.semantic_operands.destination, instruction_form.semantic_operands.src_dst, ): - if 'register' in dst: + if "register" in dst: # Check for read of register until overwrite for instr_form in kernel: if self.is_read(dst.register, instr_form): @@ -200,7 +200,7 @@ class KernelDG(nx.DiGraph): if include_write: yield instr_form break - if 'flag' in dst and flag_dependencies: + if "flag" in dst and flag_dependencies: # Check for read of flag until overwrite for instr_form in kernel: if self.is_read(dst.flag, instr_form): @@ -214,23 +214,23 @@ class KernelDG(nx.DiGraph): if include_write: yield instr_form break - elif 'memory' in dst: + elif "memory" in dst: # Check if base register is altered during memory access - if 'pre_indexed' in dst.memory or 'post_indexed' in dst.memory: + if "pre_indexed" in dst.memory or "post_indexed" in dst.memory: # Check for read of base register until overwrite for instr_form in kernel: if self.is_read(dst.memory.base, instr_form): - instr_form['mem_dep'] = instruction_form + instr_form["mem_dep"] = instruction_form yield instr_form if self.is_written(dst.memory.base, instr_form): # operand in src_dst list if include_write: - instr_form['mem_dep'] = instruction_form + instr_form["mem_dep"] = instruction_form yield instr_form break elif self.is_written(dst.memory.base, instr_form): if include_write: - instr_form['mem_dep'] = instruction_form + instr_form["mem_dep"] = instruction_form yield instr_form break @@ -239,8 +239,8 @@ class KernelDG(nx.DiGraph): Returns iterator """ if not instr_form and not line_number: - raise ValueError('Either instruction form or line_number required.') - line_number = line_number if line_number else instr_form['line_number'] + raise ValueError("Either instruction form or line_number required.") + line_number = line_number if line_number else instr_form["line_number"] if self.dg.has_node(line_number): return self.dg.successors(line_number) return iter([]) @@ -253,29 +253,25 @@ class KernelDG(nx.DiGraph): for src in chain( instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst ): - if 'register' in src: + if "register" in src: is_read = self.parser.is_reg_dependend_of(register, src.register) or is_read - if 'flag' in src: + if "flag" in src: is_read = self.parser.is_flag_dependend_of(register, src.flag) or is_read - if 'memory' in src: + if "memory" in src: if src.memory.base is not None: is_read = self.parser.is_reg_dependend_of(register, src.memory.base) or is_read if src.memory.index is not None: - is_read = ( - self.parser.is_reg_dependend_of(register, src.memory.index) or is_read - ) + is_read = self.parser.is_reg_dependend_of(register, src.memory.index) or is_read # Check also if read in destination memory address for dst in chain( instruction_form.semantic_operands.destination, instruction_form.semantic_operands.src_dst, ): - if 'memory' in dst: + if "memory" in dst: if dst.memory.base is not None: is_read = self.parser.is_reg_dependend_of(register, dst.memory.base) or is_read if dst.memory.index is not None: - is_read = ( - self.parser.is_reg_dependend_of(register, dst.memory.index) or is_read - ) + is_read = self.parser.is_reg_dependend_of(register, dst.memory.index) or is_read return is_read def is_written(self, register, instruction_form): @@ -287,12 +283,12 @@ class KernelDG(nx.DiGraph): instruction_form.semantic_operands.destination, instruction_form.semantic_operands.src_dst, ): - if 'register' in dst: + if "register" in dst: is_written = self.parser.is_reg_dependend_of(register, dst.register) or is_written - if 'flag' in dst: + if "flag" in dst: is_written = self.parser.is_flag_dependend_of(register, dst.flag) or is_written - if 'memory' in dst: - if 'pre_indexed' in dst.memory or 'post_indexed' in dst.memory: + if "memory" in dst: + if "pre_indexed" in dst.memory or "post_indexed" in dst.memory: is_written = ( self.parser.is_reg_dependend_of(register, dst.memory.base) or is_written ) @@ -300,8 +296,8 @@ class KernelDG(nx.DiGraph): for src in chain( instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst ): - if 'memory' in src: - if 'pre_indexed' in src.memory or 'post_indexed' in src.memory: + if "memory" in src: + if "pre_indexed" in src.memory or "post_indexed" in src.memory: is_written = ( self.parser.is_reg_dependend_of(register, src.memory.base) or is_written ) @@ -317,51 +313,51 @@ class KernelDG(nx.DiGraph): """ graph = copy.deepcopy(self.dg) cp = self.get_critical_path() - cp_line_numbers = [x['line_number'] for x in cp] + cp_line_numbers = [x["line_number"] for x in cp] lcd = self.get_loopcarried_dependencies() lcd_line_numbers = {} for dep in lcd: - lcd_line_numbers[dep] = [x['line_number'] for x in lcd[dep]['dependencies']] + lcd_line_numbers[dep] = [x["line_number"] for x in lcd[dep]["dependencies"]] # add color scheme - graph.graph['node'] = {'colorscheme': 'accent8'} - graph.graph['edge'] = {'colorscheme': 'accent8'} + graph.graph["node"] = {"colorscheme": "accent8"} + graph.graph["edge"] = {"colorscheme": "accent8"} # create LCD edges for dep in lcd_line_numbers: min_line_number = min(lcd_line_numbers[dep]) max_line_number = max(lcd_line_numbers[dep]) graph.add_edge(max_line_number, min_line_number) - graph.edges[max_line_number, min_line_number]['latency'] = [ - x for x in lcd[dep]['dependencies'] if x['line_number'] == max_line_number - ][0]['latency_lcd'] + graph.edges[max_line_number, min_line_number]["latency"] = [ + x for x in lcd[dep]["dependencies"] if x["line_number"] == max_line_number + ][0]["latency_lcd"] # add label to edges for e in graph.edges: - graph.edges[e]['label'] = graph.edges[e]['latency'] + graph.edges[e]["label"] = graph.edges[e]["latency"] # add CP values to graph for n in cp: - graph.nodes[n['line_number']]['instruction_form']['latency_cp'] = n['latency_cp'] + graph.nodes[n["line_number"]]["instruction_form"]["latency_cp"] = n["latency_cp"] # color CP and LCD for n in graph.nodes: if n in cp_line_numbers: # graph.nodes[n]['color'] = 1 - graph.nodes[n]['style'] = 'bold' - graph.nodes[n]['penwidth'] = 4 + graph.nodes[n]["style"] = "bold" + graph.nodes[n]["penwidth"] = 4 for col, dep in enumerate(lcd): if n in lcd_line_numbers[dep]: - if 'style' not in graph.nodes[n]: - graph.nodes[n]['style'] = 'filled' + if "style" not in graph.nodes[n]: + graph.nodes[n]["style"] = "filled" else: - graph.nodes[n]['style'] += ',filled' - graph.nodes[n]['fillcolor'] = 2 + col + graph.nodes[n]["style"] += ",filled" + graph.nodes[n]["fillcolor"] = 2 + col # color edges for e in graph.edges: if ( - graph.nodes[e[0]]['instruction_form']['line_number'] in cp_line_numbers - and graph.nodes[e[1]]['instruction_form']['line_number'] in cp_line_numbers + graph.nodes[e[0]]["instruction_form"]["line_number"] in cp_line_numbers + and graph.nodes[e[1]]["instruction_form"]["line_number"] in cp_line_numbers and e[0] < e[1] ): bold_edge = True @@ -369,38 +365,38 @@ class KernelDG(nx.DiGraph): if i in cp_line_numbers: bold_edge = False if bold_edge: - graph.edges[e]['style'] = 'bold' - graph.edges[e]['penwidth'] = 3 + graph.edges[e]["style"] = "bold" + graph.edges[e]["penwidth"] = 3 for dep in lcd_line_numbers: if ( - graph.nodes[e[0]]['instruction_form']['line_number'] in lcd_line_numbers[dep] - and graph.nodes[e[1]]['instruction_form']['line_number'] + graph.nodes[e[0]]["instruction_form"]["line_number"] in lcd_line_numbers[dep] + and graph.nodes[e[1]]["instruction_form"]["line_number"] in lcd_line_numbers[dep] ): - graph.edges[e]['color'] = graph.nodes[e[1]]['fillcolor'] + graph.edges[e]["color"] = graph.nodes[e[1]]["fillcolor"] # rename node from [idx] to [idx mnemonic] and add shape mapping = {} for n in graph.nodes: if int(n) != n: - mapping[n] = '{}: LOAD'.format(int(n)) - graph.nodes[n]['fontname'] = 'italic' - graph.nodes[n]['fontsize'] = 11.0 + mapping[n] = "{}: LOAD".format(int(n)) + graph.nodes[n]["fontname"] = "italic" + graph.nodes[n]["fontsize"] = 11.0 else: - node = graph.nodes[n]['instruction_form'] - if node['instruction'] is not None: - mapping[n] = '{}: {}'.format(n, node['instruction']) + node = graph.nodes[n]["instruction_form"] + if node["instruction"] is not None: + mapping[n] = "{}: {}".format(n, node["instruction"]) else: - label = 'label' if node['label'] else None - label = 'directive' if node['directive'] else label - label = 'comment' if node['comment'] and label is None else label - mapping[n] = '{}: {}'.format(n, label) - graph.nodes[n]['fontname'] = 'italic' - graph.nodes[n]['fontsize'] = 11.0 - graph.nodes[n]['shape'] = 'rectangle' + label = "label" if node["label"] else None + label = "directive" if node["directive"] else label + label = "comment" if node["comment"] and label is None else label + mapping[n] = "{}: {}".format(n, label) + graph.nodes[n]["fontname"] = "italic" + graph.nodes[n]["fontsize"] = 11.0 + graph.nodes[n]["shape"] = "rectangle" nx.relabel.relabel_nodes(graph, mapping, copy=False) if filepath: nx.drawing.nx_agraph.write_dot(graph, filepath) else: - nx.drawing.nx_agraph.write_dot(graph, 'osaca_dg.dot') + nx.drawing.nx_agraph.write_dot(graph, "osaca_dg.dot") diff --git a/osaca/semantics/marker_utils.py b/osaca/semantics/marker_utils.py index b9ee0c6..793e414 100755 --- a/osaca/semantics/marker_utils.py +++ b/osaca/semantics/marker_utils.py @@ -3,7 +3,7 @@ from collections import OrderedDict from osaca.parser import ParserAArch64, ParserX86ATT, get_parser -COMMENT_MARKER = {'start': 'OSACA-BEGIN', 'end': 'OSACA-END'} +COMMENT_MARKER = {"start": "OSACA-BEGIN", "end": "OSACA-END"} def reduce_to_section(kernel, isa): @@ -15,12 +15,12 @@ def reduce_to_section(kernel, isa): :returns: `list` -- marked section of kernel as list of instruction forms """ isa = isa.lower() - if isa == 'x86': + if isa == "x86": start, end = find_marked_kernel_x86ATT(kernel) - elif isa == 'aarch64': + elif isa == "aarch64": start, end = find_marked_kernel_AArch64(kernel) else: - raise ValueError('ISA not supported.') + raise ValueError("ISA not supported.") if start == -1: start = 0 if end == -1: @@ -35,12 +35,12 @@ def find_marked_kernel_AArch64(lines): :param list lines: kernel :returns: `tuple of int` -- start and end line of marked section """ - nop_bytes = ['213', '3', '32', '31'] + nop_bytes = ["213", "3", "32", "31"] return find_marked_section( lines, ParserAArch64(), - ['mov'], - 'x1', + ["mov"], + "x1", [111, 222], nop_bytes, reverse=True, @@ -55,12 +55,12 @@ def find_marked_kernel_x86ATT(lines): :param list lines: kernel :returns: `tuple of int` -- start and end line of marked section """ - nop_bytes = ['100', '103', '144'] + nop_bytes = ["100", "103", "144"] return find_marked_section( lines, ParserX86ATT(), - ['mov', 'movl'], - 'ebx', + ["mov", "movl"], + "ebx", [111, 222], nop_bytes, comments=COMMENT_MARKER, @@ -70,32 +70,32 @@ def find_marked_kernel_x86ATT(lines): def get_marker(isa, comment=""): """Return tuple of start and end marker lines.""" isa = isa.lower() - if isa == 'x86': + if isa == "x86": start_marker_raw = ( - 'movl $111, %ebx # OSACA START MARKER\n' - '.byte 100 # OSACA START MARKER\n' - '.byte 103 # OSACA START MARKER\n' - '.byte 144 # OSACA START MARKER\n' + "movl $111, %ebx # OSACA START MARKER\n" + ".byte 100 # OSACA START MARKER\n" + ".byte 103 # OSACA START MARKER\n" + ".byte 144 # OSACA START MARKER\n" ) if comment: start_marker_raw += "# {}\n".format(comment) end_marker_raw = ( - 'movl $222, %ebx # OSACA END MARKER\n' - '.byte 100 # OSACA END MARKER\n' - '.byte 103 # OSACA END MARKER\n' - '.byte 144 # OSACA END MARKER\n' + "movl $222, %ebx # OSACA END MARKER\n" + ".byte 100 # OSACA END MARKER\n" + ".byte 103 # OSACA END MARKER\n" + ".byte 144 # OSACA END MARKER\n" ) - elif isa == 'aarch64': + elif isa == "aarch64": start_marker_raw = ( - 'mov x1, #111 // OSACA START MARKER\n' - '.byte 213,3,32,31 // OSACA START MARKER\n' + "mov x1, #111 // OSACA START MARKER\n" + ".byte 213,3,32,31 // OSACA START MARKER\n" ) if comment: start_marker_raw += "// {}\n".format(comment) # After loop end_marker_raw = ( - 'mov x1, #222 // OSACA END MARKER\n' - '.byte 213,3,32,31 // OSACA END MARKER\n' + "mov x1, #222 // OSACA END MARKER\n" + ".byte 213,3,32,31 // OSACA END MARKER\n" ) parser = get_parser(isa) @@ -134,18 +134,18 @@ def find_marked_section( for i, line in enumerate(lines): try: if line.instruction is None and comments is not None and line.comment is not None: - if comments['start'] == line.comment: + if comments["start"] == line.comment: index_start = i + 1 - elif comments['end'] == line.comment: + elif comments["end"] == line.comment: index_end = i elif line.instruction in mov_instr and lines[i + 1].directive is not None: source = line.operands[0 if not reverse else 1] destination = line.operands[1 if not reverse else 0] # instruction pair matches, check for operands if ( - 'immediate' in source + "immediate" in source and parser.normalize_imd(source.immediate) == mov_vals[0] - and 'register' in destination + and "register" in destination and parser.get_full_reg_name(destination.register) == mov_reg ): # operands of first instruction match start, check for second one @@ -154,9 +154,9 @@ def find_marked_section( # return first line after the marker index_start = i + 1 + line_count elif ( - 'immediate' in source + "immediate" in source and parser.normalize_imd(source.immediate) == mov_vals[1] - and 'register' in destination + and "register" in destination and parser.get_full_reg_name(destination.register) == mov_reg ): # operand of first instruction match end, check for second one @@ -179,7 +179,7 @@ def match_bytes(lines, index, byte_list): while ( index < len(lines) and lines[index].directive is not None - and lines[index].directive.name == 'byte' + and lines[index].directive.name == "byte" ): line_count += 1 extracted_bytes += lines[index].directive.parameters @@ -199,14 +199,14 @@ def find_jump_labels(lines): labels = OrderedDict() current_label = None for i, line in enumerate(lines): - if line['label'] is not None: + if line["label"] is not None: # When a new label is found, add to blocks dict - labels[line['label']] = (i,) + labels[line["label"]] = (i,) # End previous block at previous line if current_label is not None: labels[current_label] = (labels[current_label][0], i) # Update current block name - current_label = line['label'] + current_label = line["label"] elif current_label is None: # If no block has been started, skip end detection continue @@ -218,9 +218,9 @@ def find_jump_labels(lines): for label in list(labels): if all( [ - l['instruction'].startswith('.') + l["instruction"].startswith(".") for l in lines[labels[label][0] : labels[label][1]] - if l['instruction'] is not None + if l["instruction"] is not None ] ): del labels[label] @@ -247,11 +247,11 @@ def find_basic_blocks(lines): terminate = False blocks[label].append(line) # Find end of block by searching for references to valid jump labels - if line['instruction'] and line['operands']: - for operand in [o for o in line['operands'] if 'identifier' in o]: - if operand['identifier']['name'] in valid_jump_labels: + if line["instruction"] and line["operands"]: + for operand in [o for o in line["operands"] if "identifier" in o]: + if operand["identifier"]["name"] in valid_jump_labels: terminate = True - elif line['label'] is not None: + elif line["label"] is not None: terminate = True if terminate: break @@ -276,15 +276,15 @@ def find_basic_loop_bodies(lines): terminate = False current_block.append(line) # Find end of block by searching for references to valid jump labels - if line['instruction'] and line['operands']: + if line["instruction"] and line["operands"]: # Ignore `b.none` instructions (relevant von ARM SVE code) - # This branch instruction is often present _within_ inner loop blocks, but usually + # This branch instruction is often present _within_ inner loop blocks, but usually # do not terminate - if line['instruction'] == 'b.none': + if line["instruction"] == "b.none": continue - for operand in [o for o in line['operands'] if 'identifier' in o]: - if operand['identifier']['name'] in valid_jump_labels: - if operand['identifier']['name'] == label: + for operand in [o for o in line["operands"] if "identifier" in o]: + if operand["identifier"]["name"] in valid_jump_labels: + if operand["identifier"]["name"] == label: loop_bodies[label] = current_block terminate = True break diff --git a/osaca/utils.py b/osaca/utils.py index 53ff292..c235534 100644 --- a/osaca/utils.py +++ b/osaca/utils.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import os.path -DATA_DIRS = [os.path.expanduser('~/.osaca/data'), os.path.join(os.path.dirname(__file__), 'data')] -CACHE_DIR = os.path.expanduser('~/.osaca/cache') +DATA_DIRS = [os.path.expanduser("~/.osaca/data"), os.path.join(os.path.dirname(__file__), "data")] +CACHE_DIR = os.path.expanduser("~/.osaca/cache") def find_datafile(name): diff --git a/setup.py b/setup.py index 38785b0..e72d930 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from setuptools import setup, find_packages from setuptools.command.install import install as _install from setuptools.command.sdist import sdist as _sdist + # To use a consistent encoding from codecs import open import os @@ -16,8 +17,9 @@ here = os.path.abspath(os.path.dirname(__file__)) # Stolen from pip def read(*names, **kwargs): - with io.open(os.path.join(os.path.dirname(__file__), *names), - encoding=kwargs.get("encoding", "utf8")) as fp: + with io.open( + os.path.join(os.path.dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8") + ) as fp: return fp.read() @@ -32,11 +34,11 @@ def find_version(*file_paths): def _run_build_cache(dir): from subprocess import check_call + # This is run inside the install staging directory (that had no .pyc files) # We don't want to generate any. # https://github.com/eliben/pycparser/pull/135 - check_call([sys.executable, '-B', '_build_cache.py'], - cwd=os.path.join(dir, 'osaca', 'data')) + check_call([sys.executable, "-B", "_build_cache.py"], cwd=os.path.join(dir, "osaca", "data")) class install(_install): @@ -52,77 +54,60 @@ class sdist(_sdist): # Get the long description from the README file -with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: +with open(os.path.join(here, "README.rst"), encoding="utf-8") as f: long_description = f.read() setup( - name='osaca', - + name="osaca", # Version should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/distributing.html - version=find_version('osaca', '__init__.py'), - - description='Open Source Architecture Code Analyzer', + version=find_version("osaca", "__init__.py"), + description="Open Source Architecture Code Analyzer", long_description=long_description, - long_description_content_type='text/x-rst', - + long_description_content_type="text/x-rst", # The project's main homepage - url='https://github.com/RRZE-HPC/OSACA', - + url="https://github.com/RRZE-HPC/OSACA", # Author details - author='Jan Laukemann', - author_email='jan.laukemann@fau.de', - + author="Jan Laukemann", + author_email="jan.laukemann@fau.de", # Choose your license - license='AGPLv3', - + license="AGPLv3", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ # How mature is this project? Common values are # 3 - Alpha # 4 - Beta # 5 - Production/Stable - 'Development Status :: 4 - Beta', - + "Development Status :: 4 - Beta", # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Topic :: Scientific/Engineering', - 'Topic :: Software Development', - 'Topic :: Utilities', - + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", + "Topic :: Utilities", # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: GNU Affero General Public License v3', - + "License :: OSI Approved :: GNU Affero General Public License v3", # Specify the Python versions you support here. In particular, ensure # that you indicate wheter you support Python2, Python 3 or both. - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], - # What doesd your project relate to? - keywords='hpc performance benchmark analysis architecture', - + keywords="hpc performance benchmark analysis architecture", # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - + packages=find_packages(exclude=["contrib", "docs", "tests"]), # List run-time dependencies here. These will be installed by pip when # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html - install_requires=[ - 'networkx', - 'pyparsing>=2.3.1', - 'ruamel.yaml>=0.15.71', - ], - python_requires='>=3.5', - + install_requires=["networkx", "pyparsing>=2.3.1", "ruamel.yaml>=0.15.71",], + python_requires=">=3.5", # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: @@ -131,27 +116,19 @@ setup( # 'dev': ['check-manifest'], # 'test': ['coverage'], # }, - # If there are data files included in your packages that need to be # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. include_package_data=True, - # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa # In this case, 'data_file' will be installed into '/my_data' # data_files=[('my_data', ['data/data_file'])], - # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow # pip to create the appropriate form of executable for the target platform. - entry_points={ - 'console_scripts': [ - 'osaca=osaca.osaca:main', - ], - }, - + entry_points={"console_scripts": ["osaca=osaca.osaca:main",],}, # Overwriting install and sdist to enforce cache distribution with package - cmdclass={'install': install, 'sdist': sdist}, + cmdclass={"install": install, "sdist": sdist}, ) diff --git a/tests/all_tests.py b/tests/all_tests.py index ac7b47f..c4d6773 100755 --- a/tests/all_tests.py +++ b/tests/all_tests.py @@ -3,18 +3,18 @@ import sys import unittest -sys.path[0:0] = ['.', '..'] +sys.path[0:0] = [".", ".."] suite = unittest.TestLoader().loadTestsFromNames( [ - 'test_base_parser', - 'test_parser_x86att', - 'test_parser_AArch64', - 'test_marker_utils', - 'test_semantics', - 'test_frontend', - 'test_db_interface', - 'test_kerncraftAPI', - 'test_cli', + "test_base_parser", + "test_parser_x86att", + "test_parser_AArch64", + "test_marker_utils", + "test_semantics", + "test_frontend", + "test_db_interface", + "test_kerncraftAPI", + "test_cli", ] ) diff --git a/tests/test_base_parser.py b/tests/test_base_parser.py index 4d76352..77f92d0 100755 --- a/tests/test_base_parser.py +++ b/tests/test_base_parser.py @@ -16,13 +16,13 @@ class TestBaseParser(unittest.TestCase): self.parser = BaseParser() except NotImplementedError: pass - with open(self._find_file('triad_x86_iaca.s')) as f: + with open(self._find_file("triad_x86_iaca.s")) as f: self.triad_code = f.read() - with open(self._find_file('triad_arm_iaca.s')) as f: + with open(self._find_file("triad_arm_iaca.s")) as f: self.triad_code_arm = f.read() - with open(self._find_file('kernel_x86.s')) as f: + with open(self._find_file("kernel_x86.s")) as f: self.x86_code = f.read() - with open(self._find_file('kernel_aarch64.s')) as f: + with open(self._find_file("kernel_aarch64.s")) as f: self.aarch64_code = f.read() ################## @@ -34,19 +34,19 @@ class TestBaseParser(unittest.TestCase): self.parser.parse_file(self.triad_code) def test_parse_line(self): - line_instruction = '\t\tlea 2(%rax,%rax), %ecx #12.9' + line_instruction = "\t\tlea 2(%rax,%rax), %ecx #12.9" with self.assertRaises(NotImplementedError): self.parser.parse_line(line_instruction) def test_parse_instruction(self): - instr1 = '\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27' + instr1 = "\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27" with self.assertRaises(NotImplementedError): self.parser.parse_instruction(instr1) def test_register_funcs(self): - reg_a1 = AttrDict({'name': 'rax'}) - reg_a2 = AttrDict({'name': 'eax'}) - register_string = 'v1.2d' + reg_a1 = AttrDict({"name": "rax"}) + reg_a2 = AttrDict({"name": "eax"}) + register_string = "v1.2d" with self.assertRaises(NotImplementedError): self.parser.is_reg_dependend_of(reg_a1, reg_a2) with self.assertRaises(NotImplementedError): @@ -61,15 +61,15 @@ class TestBaseParser(unittest.TestCase): self.parser.get_full_reg_name(reg_a1) def test_normalize_imd(self): - imd_hex_1 = {'value': '0x4f'} + imd_hex_1 = {"value": "0x4f"} with self.assertRaises(NotImplementedError): self.parser.normalize_imd(imd_hex_1) def test_detect_ISA(self): - self.assertEqual(BaseParser.detect_ISA(self.triad_code), 'x86') - self.assertEqual(BaseParser.detect_ISA(self.triad_code_arm), 'aarch64') - self.assertEqual(BaseParser.detect_ISA(self.x86_code), 'x86') - self.assertEqual(BaseParser.detect_ISA(self.aarch64_code), 'aarch64') + self.assertEqual(BaseParser.detect_ISA(self.triad_code), "x86") + self.assertEqual(BaseParser.detect_ISA(self.triad_code_arm), "aarch64") + self.assertEqual(BaseParser.detect_ISA(self.x86_code), "x86") + self.assertEqual(BaseParser.detect_ISA(self.aarch64_code), "aarch64") ################## # Helper functions @@ -78,11 +78,11 @@ class TestBaseParser(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestBaseParser) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_cli.py b/tests/test_cli.py index 3141cac..1669bbd 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -29,11 +29,11 @@ class TestCLI(unittest.TestCase): def test_check_arguments(self): parser = osaca.create_parser(parser=ErrorRaisingArgumentParser()) - args = parser.parse_args(['--arch', 'WRONG_ARCH', self._find_file('gs', 'csx', 'gcc')]) + args = parser.parse_args(["--arch", "WRONG_ARCH", self._find_file("gs", "csx", "gcc")]) with self.assertRaises(ValueError): osaca.check_arguments(args, parser) args = parser.parse_args( - ['--arch', 'csx', '--import', 'WRONG_BENCH', self._find_file('gs', 'csx', 'gcc')] + ["--arch", "csx", "--import", "WRONG_BENCH", self._find_file("gs", "csx", "gcc")] ) with self.assertRaises(ValueError): osaca.check_arguments(args, parser) @@ -42,22 +42,22 @@ class TestCLI(unittest.TestCase): parser = osaca.create_parser(parser=ErrorRaisingArgumentParser()) args = parser.parse_args( [ - '--arch', - 'tx2', - '--import', - 'ibench', - self._find_test_file('ibench_import_aarch64.dat'), + "--arch", + "tx2", + "--import", + "ibench", + self._find_test_file("ibench_import_aarch64.dat"), ] ) output = StringIO() osaca.run(args, output_file=output) args = parser.parse_args( [ - '--arch', - 'tx2', - '--import', - 'asmbench', - self._find_test_file('asmbench_import_aarch64.dat'), + "--arch", + "tx2", + "--import", + "asmbench", + self._find_test_file("asmbench_import_aarch64.dat"), ] ) osaca.run(args, output_file=output) @@ -65,28 +65,28 @@ class TestCLI(unittest.TestCase): def test_check_db(self): parser = osaca.create_parser(parser=ErrorRaisingArgumentParser()) args = parser.parse_args( - ['--arch', 'tx2', '--db-check', '--verbose', self._find_test_file('triad_x86_iaca.s')] + ["--arch", "tx2", "--db-check", "--verbose", self._find_test_file("triad_x86_iaca.s")] ) output = StringIO() osaca.run(args, output_file=output) def test_get_parser(self): - self.assertTrue(isinstance(osaca.get_asm_parser('csx'), ParserX86ATT)) - self.assertTrue(isinstance(osaca.get_asm_parser('tx2'), ParserAArch64)) + self.assertTrue(isinstance(osaca.get_asm_parser("csx"), ParserX86ATT)) + self.assertTrue(isinstance(osaca.get_asm_parser("tx2"), ParserAArch64)) with self.assertRaises(ValueError): - osaca.get_asm_parser('UNKNOWN') + osaca.get_asm_parser("UNKNOWN") def test_marker_insert_x86(self): # copy file to add markers - name = self._find_test_file('kernel_x86.s') - name_copy = name + '.copy.s' + name = self._find_test_file("kernel_x86.s") + name_copy = name + ".copy.s" copyfile(name, name_copy) - user_input = ['.L10'] + user_input = [".L10"] output = StringIO() parser = osaca.create_parser() - args = parser.parse_args(['--arch', 'csx', '--insert-marker', name_copy]) - with patch('builtins.input', side_effect=user_input): + args = parser.parse_args(["--arch", "csx", "--insert-marker", name_copy]) + with patch("builtins.input", side_effect=user_input): osaca.run(args, output_file=output) lines_orig = len(open(name).readlines()) @@ -97,14 +97,14 @@ class TestCLI(unittest.TestCase): def test_marker_insert_aarch64(self): # copy file to add markers - name = self._find_test_file('kernel_aarch64.s') - name_copy = name + '.copy.s' + name = self._find_test_file("kernel_aarch64.s") + name_copy = name + ".copy.s" copyfile(name, name_copy) - user_input = ['.LBB0_32', '64'] + user_input = [".LBB0_32", "64"] parser = osaca.create_parser() - args = parser.parse_args(['--arch', 'tx2', '--insert-marker', name_copy]) - with patch('builtins.input', side_effect=user_input): + args = parser.parse_args(["--arch", "tx2", "--insert-marker", name_copy]) + with patch("builtins.input", side_effect=user_input): osaca.run(args) lines_orig = len(open(name).readlines()) @@ -115,18 +115,18 @@ class TestCLI(unittest.TestCase): def test_examples(self): kernels = [ - 'add', - 'copy', - 'daxpy', - 'gs', - 'j2d', - 'striad', - 'sum_reduction', - 'triad', - 'update', + "add", + "copy", + "daxpy", + "gs", + "j2d", + "striad", + "sum_reduction", + "triad", + "update", ] - archs = ['csx', 'tx2', 'zen1'] - comps = {'csx': ['gcc', 'icc'], 'tx2': ['gcc', 'clang'], 'zen1': ['gcc']} + archs = ["csx", "tx2", "zen1"] + comps = {"csx": ["gcc", "icc"], "tx2": ["gcc", "clang"], "zen1": ["gcc"]} parser = osaca.create_parser() # Analyze all asm files resulting out of kernels, archs and comps for k in kernels: @@ -134,11 +134,11 @@ class TestCLI(unittest.TestCase): for c in comps[a]: with self.subTest(kernel=k, arch=a, comp=c): args = parser.parse_args( - ['--arch', a, self._find_file(k, a, c), '--export-graph', '/dev/null'] + ["--arch", a, self._find_file(k, a, c), "--export-graph", "/dev/null"] ) output = StringIO() osaca.run(args, output_file=output) - self.assertTrue('WARNING' not in output.getvalue()) + self.assertTrue("WARNING" not in output.getvalue()) def test_architectures(self): parser = osaca.create_parser() @@ -147,10 +147,8 @@ class TestCLI(unittest.TestCase): for arch in archs: with self.subTest(micro_arch=arch): isa = MachineModel.get_isa_for_arch(arch) - kernel = 'kernel_{}.s'.format(isa) - args = parser.parse_args( - ['--arch', arch, self._find_test_file(kernel)] - ) + kernel = "kernel_{}.s".format(isa) + args = parser.parse_args(["--arch", arch, self._find_test_file(kernel)]) output = StringIO() osaca.run(args, output_file=output) @@ -168,59 +166,68 @@ class TestCLI(unittest.TestCase): # Run test kernels without --arch flag parser = osaca.create_parser() # x86 - kernel_x86 = 'kernel_x86.s' + kernel_x86 = "kernel_x86.s" args = parser.parse_args([self._find_test_file(kernel_x86)]) output = StringIO() osaca.run(args, output_file=output) # AArch64 - kernel_aarch64 = 'kernel_aarch64.s' + kernel_aarch64 = "kernel_aarch64.s" args = parser.parse_args([self._find_test_file(kernel_aarch64)]) osaca.run(args, output_file=output) - + def test_user_warnings(self): parser = osaca.create_parser() - kernel = 'triad_x86_unmarked.s' + kernel = "triad_x86_unmarked.s" args = parser.parse_args( - ['--arch', 'csx', '--ignore-unknown', self._find_test_file(kernel)] + ["--arch", "csx", "--ignore-unknown", self._find_test_file(kernel)] ) output = StringIO() osaca.run(args, output_file=output) # WARNING for length - self.assertTrue(output.getvalue().count('WARNING') == 1) + self.assertTrue(output.getvalue().count("WARNING") == 1) args = parser.parse_args( - ['--lines', '100-199', '--ignore-unknown', self._find_test_file(kernel)] + ["--lines", "100-199", "--ignore-unknown", self._find_test_file(kernel)] ) output = StringIO() osaca.run(args, output_file=output) # WARNING for arch - self.assertTrue(output.getvalue().count('WARNING') == 1) - + self.assertTrue(output.getvalue().count("WARNING") == 1) def test_lines_arg(self): # Run tests with --lines option parser = osaca.create_parser() - kernel_x86 = 'triad_x86_iaca.s' - args_base = parser.parse_args( - ['--arch', 'csx', self._find_test_file(kernel_x86)] - ) + kernel_x86 = "triad_x86_iaca.s" + args_base = parser.parse_args(["--arch", "csx", self._find_test_file(kernel_x86)]) output_base = StringIO() osaca.run(args_base, output_file=output_base) - output_base = output_base.getvalue().split('\n')[8:] + output_base = output_base.getvalue().split("\n")[8:] args = [] - args.append(parser.parse_args( - ['--lines', '146-154', '--arch', 'csx', self._find_test_file(kernel_x86)] - )) - args.append(parser.parse_args( - ['--lines', '146:154', '--arch', 'csx', self._find_test_file(kernel_x86)] - )) - args.append(parser.parse_args( - ['--lines', '146,147:148,149-154', '--arch', 'csx', self._find_test_file(kernel_x86)] - )) + args.append( + parser.parse_args( + ["--lines", "146-154", "--arch", "csx", self._find_test_file(kernel_x86)] + ) + ) + args.append( + parser.parse_args( + ["--lines", "146:154", "--arch", "csx", self._find_test_file(kernel_x86)] + ) + ) + args.append( + parser.parse_args( + [ + "--lines", + "146,147:148,149-154", + "--arch", + "csx", + self._find_test_file(kernel_x86), + ] + ) + ) for a in args: with self.subTest(params=a): output = StringIO() osaca.run(a, output_file=output) - self.assertEqual(output.getvalue().split('\n')[8:], output_base) + self.assertEqual(output.getvalue().split("\n")[8:], output_base) ################## # Helper functions @@ -231,23 +238,23 @@ class TestCLI(unittest.TestCase): testdir = os.path.dirname(__file__) name = os.path.join( testdir, - '../examples', + "../examples", kernel, - kernel + '.s.' + arch[:3].lower() + '.' + comp.lower() + '.s', + kernel + ".s." + arch[:3].lower() + "." + comp.lower() + ".s", ) - if kernel == 'j2d' and arch.lower() == 'csx': - name = name[:-1] + 'AVX.s' + if kernel == "j2d" and arch.lower() == "csx": + name = name[:-1] + "AVX.s" assert os.path.exists(name) return name @staticmethod def _find_test_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestCLI) unittest.TextTestRunner(verbosity=2, buffer=True).run(suite) diff --git a/tests/test_db_interface.py b/tests/test_db_interface.py index 800c610..7678ad0 100755 --- a/tests/test_db_interface.py +++ b/tests/test_db_interface.py @@ -15,47 +15,47 @@ class TestDBInterface(unittest.TestCase): @classmethod def setUpClass(self): sample_entry = { - 'name': 'DoItRightAndDoItFast', - 'operands': [ - {'class': 'memory', 'offset': 'imd', 'base': 'gpr', 'index': 'gpr', 'scale': 8}, - {'class': 'register', 'name': 'xmm'}, + "name": "DoItRightAndDoItFast", + "operands": [ + {"class": "memory", "offset": "imd", "base": "gpr", "index": "gpr", "scale": 8}, + {"class": "register", "name": "xmm"}, ], - 'throughput': 1.25, - 'latency': 125, - 'uops': 6, + "throughput": 1.25, + "latency": 125, + "uops": 6, } self.entry_csx = sample_entry.copy() self.entry_tx2 = sample_entry.copy() self.entry_zen1 = sample_entry.copy() # self.entry_csx['port_pressure'] = [1.25, 0, 1.25, 0.5, 0.5, 0.5, 0.5, 0, 1.25, 1.25, 0] - self.entry_csx['port_pressure'] = [[5, '0156'], [1, '23'], [1, ['2D', '3D']]] + self.entry_csx["port_pressure"] = [[5, "0156"], [1, "23"], [1, ["2D", "3D"]]] # self.entry_tx2['port_pressure'] = [2.5, 2.5, 0, 0, 0.5, 0.5] - self.entry_tx2['port_pressure'] = [[5, '01'], [1, '45']] - del self.entry_tx2['operands'][1]['name'] - self.entry_tx2['operands'][1]['prefix'] = 'x' + self.entry_tx2["port_pressure"] = [[5, "01"], [1, "45"]] + del self.entry_tx2["operands"][1]["name"] + self.entry_tx2["operands"][1]["prefix"] = "x" # self.entry_zen1['port_pressure'] = [1, 1, 1, 1, 0, 1, 0, 0, 0, 0.5, 1, 0.5, 1] - self.entry_zen1['port_pressure'] = [[4, '0123'], [1, '4'], [1, '89'], [2, ['8D', '9D']]] + self.entry_zen1["port_pressure"] = [[4, "0123"], [1, "4"], [1, "89"], [2, ["8D", "9D"]]] ########### # Tests ########### def test_add_single_entry(self): - mm_csx = MachineModel('csx') - mm_tx2 = MachineModel('tx2') - mm_zen1 = MachineModel('zen1') - num_entries_csx = len(mm_csx['instruction_forms']) - num_entries_tx2 = len(mm_tx2['instruction_forms']) - num_entries_zen1 = len(mm_zen1['instruction_forms']) + mm_csx = MachineModel("csx") + mm_tx2 = MachineModel("tx2") + mm_zen1 = MachineModel("zen1") + num_entries_csx = len(mm_csx["instruction_forms"]) + num_entries_tx2 = len(mm_tx2["instruction_forms"]) + num_entries_zen1 = len(mm_zen1["instruction_forms"]) mm_csx.set_instruction_entry(self.entry_csx) mm_tx2.set_instruction_entry(self.entry_tx2) - mm_zen1.set_instruction_entry({'name': 'empty_operation'}) + mm_zen1.set_instruction_entry({"name": "empty_operation"}) - num_entries_csx = len(mm_csx['instruction_forms']) - num_entries_csx - num_entries_tx2 = len(mm_tx2['instruction_forms']) - num_entries_tx2 - num_entries_zen1 = len(mm_zen1['instruction_forms']) - num_entries_zen1 + num_entries_csx = len(mm_csx["instruction_forms"]) - num_entries_csx + num_entries_tx2 = len(mm_tx2["instruction_forms"]) - num_entries_tx2 + num_entries_zen1 = len(mm_zen1["instruction_forms"]) - num_entries_zen1 self.assertEqual(num_entries_csx, 1) self.assertEqual(num_entries_tx2, 1) @@ -64,76 +64,76 @@ class TestDBInterface(unittest.TestCase): def test_invalid_add(self): entry = {} with self.assertRaises(KeyError): - MachineModel('csx').set_instruction_entry(entry) + MachineModel("csx").set_instruction_entry(entry) with self.assertRaises(TypeError): - MachineModel('csx').set_instruction() + MachineModel("csx").set_instruction() def test_sanity_check(self): output = StringIO() # non-verbose - sanity_check('csx', verbose=False, internet_check=False, output_file=output) - sanity_check('tx2', verbose=False, internet_check=False, output_file=output) - sanity_check('zen1', verbose=False, internet_check=False, output_file=output) + sanity_check("csx", verbose=False, internet_check=False, output_file=output) + sanity_check("tx2", verbose=False, internet_check=False, output_file=output) + sanity_check("zen1", verbose=False, internet_check=False, output_file=output) # verbose - sanity_check('csx', verbose=True, internet_check=False, output_file=output) - sanity_check('tx2', verbose=True, internet_check=False, output_file=output) - sanity_check('zen1', verbose=True, internet_check=False, output_file=output) + sanity_check("csx", verbose=True, internet_check=False, output_file=output) + sanity_check("tx2", verbose=True, internet_check=False, output_file=output) + sanity_check("zen1", verbose=True, internet_check=False, output_file=output) def test_ibench_import(self): # only check import without dumping the DB file (takes too much time) - with open(self._find_file('ibench_import_x86.dat')) as input_file: + with open(self._find_file("ibench_import_x86.dat")) as input_file: input_data = input_file.readlines() - entries = dbi._get_ibench_output(input_data, 'x86') + entries = dbi._get_ibench_output(input_data, "x86") self.assertEqual(len(entries), 3) for _, e in entries.items(): - self.assertIsNotNone(e['throughput']) - self.assertIsNotNone(e['latency']) - with open(self._find_file('ibench_import_aarch64.dat')) as input_file: + self.assertIsNotNone(e["throughput"]) + self.assertIsNotNone(e["latency"]) + with open(self._find_file("ibench_import_aarch64.dat")) as input_file: input_data = input_file.readlines() - entries = dbi._get_ibench_output(input_data, 'aarch64') + entries = dbi._get_ibench_output(input_data, "aarch64") self.assertEqual(len(entries), 4) for _, e in entries.items(): - self.assertIsNotNone(e['throughput']) - self.assertIsNotNone(e['latency']) + self.assertIsNotNone(e["throughput"]) + self.assertIsNotNone(e["latency"]) def test_asmbench_import(self): # only check import without dumping the DB file (takes too much time) - with open(self._find_file('asmbench_import_x86.dat')) as input_file: + with open(self._find_file("asmbench_import_x86.dat")) as input_file: input_data = input_file.readlines() - entries = dbi._get_asmbench_output(input_data, 'x86') + entries = dbi._get_asmbench_output(input_data, "x86") self.assertEqual(len(entries), 3) for _, e in entries.items(): - self.assertIsNotNone(e['throughput']) - self.assertIsNotNone(e['latency']) - with open(self._find_file('asmbench_import_aarch64.dat')) as input_file: + self.assertIsNotNone(e["throughput"]) + self.assertIsNotNone(e["latency"]) + with open(self._find_file("asmbench_import_aarch64.dat")) as input_file: input_data = input_file.readlines() - entries = dbi._get_asmbench_output(input_data, 'aarch64') + entries = dbi._get_asmbench_output(input_data, "aarch64") self.assertEqual(len(entries), 4) for _, e in entries.items(): - self.assertIsNotNone(e['throughput']) - self.assertIsNotNone(e['latency']) + self.assertIsNotNone(e["throughput"]) + self.assertIsNotNone(e["latency"]) # remove empty line => no import since broken format del input_data[3] - entries = dbi._get_asmbench_output(input_data, 'aarch64') + entries = dbi._get_asmbench_output(input_data, "aarch64") self.assertEqual(len(entries), 0) with self.assertRaises(ValueError): dbi.import_benchmark_output( - 'csx', 'invalid_bench_type', self._find_file('asmbench_import_x86.dat') + "csx", "invalid_bench_type", self._find_file("asmbench_import_x86.dat") ) with self.assertRaises(AssertionError): - dbi.import_benchmark_output('csx', 'ibench', 'invalid_file') + dbi.import_benchmark_output("csx", "ibench", "invalid_file") def test_online_scraping(self): # addpd -- suspicious instruction, normal URL - instr_1 = ['addpd', (True, '(r) (r,w)')] + instr_1 = ["addpd", (True, "(r) (r,w)")] self.assertEqual(dbi._scrape_from_felixcloutier(instr_1[0]), instr_1[1]) # movpd -- not suspicious, - instr_2 = ['movapd', (False, '(r) (w)')] + instr_2 = ["movapd", (False, "(r) (w)")] self.assertEqual(dbi._scrape_from_felixcloutier(instr_2[0]), instr_2[1]) # vfmadd132pd -- only in combined view with 213/231. # No 2-operand version, therefore, empty string - instr_3 = ['vfmadd132pd', (True, '')] + instr_3 = ["vfmadd132pd", (True, "")] self.assertEqual(dbi._scrape_from_felixcloutier(instr_3[0]), instr_3[1]) ################## @@ -142,11 +142,11 @@ class TestDBInterface(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestDBInterface) unittest.TextTestRunner(verbosity=2, buffer=True).run(suite) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 9c2ef46..c72ebaf 100755 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -13,7 +13,7 @@ from osaca.semantics import ArchSemantics, KernelDG, MachineModel class TestFrontend(unittest.TestCase): MODULE_DATA_DIR = os.path.join( - os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' + os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), "osaca/data/" ) @classmethod @@ -21,26 +21,24 @@ class TestFrontend(unittest.TestCase): # set up parser and kernels self.parser_x86 = ParserX86ATT() self.parser_AArch64 = ParserAArch64() - with open(self._find_file('kernel_x86.s')) as f: + with open(self._find_file("kernel_x86.s")) as f: code_x86 = f.read() - with open(self._find_file('kernel_aarch64.s')) as f: + with open(self._find_file("kernel_aarch64.s")) as f: code_AArch64 = f.read() self.kernel_x86 = self.parser_x86.parse_file(code_x86) self.kernel_AArch64 = self.parser_AArch64.parse_file(code_AArch64) # set up machine models self.machine_model_csx = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') - ) - self.machine_model_tx2 = MachineModel( - arch='tx2' + path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml") ) + self.machine_model_tx2 = MachineModel(arch="tx2") self.semantics_csx = ArchSemantics( - self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') + self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "isa/x86.yml") ) self.semantics_tx2 = ArchSemantics( self.machine_model_tx2, - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'), + path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "isa/aarch64.yml"), ) for i in range(len(self.kernel_x86)): self.semantics_csx.assign_src_dst(self.kernel_x86[i]) @@ -57,23 +55,23 @@ class TestFrontend(unittest.TestCase): with self.assertRaises(ValueError): Frontend() with self.assertRaises(ValueError): - Frontend(arch='csx', path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')) + Frontend(arch="csx", path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml")) with self.assertRaises(FileNotFoundError): - Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'THE_MACHINE.yml')) + Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "THE_MACHINE.yml")) with self.assertRaises(FileNotFoundError): - Frontend(arch='THE_MACHINE') - Frontend(arch='zen1') + Frontend(arch="THE_MACHINE") + Frontend(arch="zen1") def test_frontend_x86(self): dg = KernelDG(self.kernel_x86, self.parser_x86, self.machine_model_csx) - fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')) + fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml")) fe.throughput_analysis(self.kernel_x86, show_cmnts=False) fe.latency_analysis(dg.get_critical_path()) # TODO compare output with checked string def test_frontend_AArch64(self): dg = KernelDG(self.kernel_AArch64, self.parser_AArch64, self.machine_model_tx2) - fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml')) + fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "tx2.yml")) fe.full_analysis(self.kernel_AArch64, dg, verbose=True) # TODO compare output with checked string @@ -84,11 +82,11 @@ class TestFrontend(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestFrontend) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_kerncraftAPI.py b/tests/test_kerncraftAPI.py index c6876e0..679a4b8 100755 --- a/tests/test_kerncraftAPI.py +++ b/tests/test_kerncraftAPI.py @@ -18,9 +18,9 @@ class TestKerncraftAPI(unittest.TestCase): # set up parser and kernels self.parser_x86 = ParserX86ATT() self.parser_AArch64 = ParserAArch64() - with open(self._find_file('triad_x86_iaca.s')) as f: + with open(self._find_file("triad_x86_iaca.s")) as f: self.code_x86 = f.read() - with open(self._find_file('triad_arm_iaca.s')) as f: + with open(self._find_file("triad_arm_iaca.s")) as f: self.code_AArch64 = f.read() ########### @@ -28,23 +28,23 @@ class TestKerncraftAPI(unittest.TestCase): ########### def test_kerncraft_API_x86(self): - kapi = KerncraftAPI('csx', self.code_x86) + kapi = KerncraftAPI("csx", self.code_x86) kapi.create_output() self.assertEqual(kapi.get_unmatched_instruction_ratio(), 0.0) port_occupation = OrderedDict( [ - ('0', 1.25), - ('0DV', 0.0), - ('1', 1.25), - ('2', 2.0), - ('2D', 1.5), - ('3', 2.0), - ('3D', 1.5), - ('4', 1.0), - ('5', 0.75), - ('6', 0.75), - ('7', 0.0), + ("0", 1.25), + ("0DV", 0.0), + ("1", 1.25), + ("2", 2.0), + ("2D", 1.5), + ("3", 2.0), + ("3D", 1.5), + ("4", 1.0), + ("5", 0.75), + ("6", 0.75), + ("7", 0.0), ] ) self.assertEqual(kapi.get_port_occupation_cycles(), port_occupation) @@ -53,20 +53,20 @@ class TestKerncraftAPI(unittest.TestCase): self.assertEqual(kapi.get_latency(), (1.0, 8.0)) def test_kerncraft_API_AArch64(self): - kapi = KerncraftAPI('tx2', self.code_AArch64) + kapi = KerncraftAPI("tx2", self.code_AArch64) kapi.create_output() self.assertEqual(kapi.get_unmatched_instruction_ratio(), 0.0) port_occupation = OrderedDict( [ - ('0', 34.0), - ('0DV', 0.0), - ('1', 34.0), - ('1DV', 0.0), - ('2', 3.0), - ('3', 64.0), - ('4', 64.0), - ('5', 32.0), + ("0", 34.0), + ("0DV", 0.0), + ("1", 34.0), + ("1DV", 0.0), + ("2", 3.0), + ("3", 64.0), + ("4", 64.0), + ("5", 32.0), ] ) self.assertEqual(kapi.get_port_occupation_cycles(), port_occupation) @@ -81,11 +81,11 @@ class TestKerncraftAPI(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestKerncraftAPI) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_marker_utils.py b/tests/test_marker_utils.py index 845c59b..5d38324 100755 --- a/tests/test_marker_utils.py +++ b/tests/test_marker_utils.py @@ -6,8 +6,12 @@ import os import unittest from collections import OrderedDict -from osaca.semantics import reduce_to_section, find_basic_blocks, find_jump_labels, \ - find_basic_loop_bodies +from osaca.semantics import ( + reduce_to_section, + find_basic_blocks, + find_jump_labels, + find_basic_loop_bodies, +) from osaca.parser import ParserAArch64, ParserX86ATT @@ -16,9 +20,9 @@ class TestMarkerUtils(unittest.TestCase): def setUpClass(self): self.parser_AArch = ParserAArch64() self.parser_x86 = ParserX86ATT() - with open(self._find_file('triad_arm_iaca.s')) as f: + with open(self._find_file("triad_arm_iaca.s")) as f: triad_code_arm = f.read() - with open(self._find_file('triad_x86_iaca.s')) as f: + with open(self._find_file("triad_x86_iaca.s")) as f: triad_code_x86 = f.read() self.parsed_AArch = self.parser_AArch.parse_file(triad_code_arm) self.parsed_x86 = self.parser_x86.parse_file(triad_code_x86) @@ -28,44 +32,44 @@ class TestMarkerUtils(unittest.TestCase): ################# def test_marker_detection_AArch64(self): - kernel = reduce_to_section(self.parsed_AArch, 'AArch64') + kernel = reduce_to_section(self.parsed_AArch, "AArch64") self.assertEqual(len(kernel), 138) self.assertEqual(kernel[0].line_number, 307) self.assertEqual(kernel[-1].line_number, 444) def test_marker_detection_x86(self): - kernel = reduce_to_section(self.parsed_x86, 'x86') + kernel = reduce_to_section(self.parsed_x86, "x86") self.assertEqual(len(kernel), 9) self.assertEqual(kernel[0].line_number, 146) self.assertEqual(kernel[-1].line_number, 154) def test_marker_matching_AArch64(self): # preparation - bytes_1_line = '.byte 213,3,32,31\n' - bytes_2_lines_1 = '.byte 213,3,32\n' + '.byte 31\n' - bytes_2_lines_2 = '.byte 213,3\n' + '.byte 32,31\n' - bytes_2_lines_3 = '.byte 213\n' + '.byte 3,32,31\n' - bytes_3_lines_1 = '.byte 213,3\n' + '.byte 32\n' + '.byte 31\n' - bytes_3_lines_2 = '.byte 213\n' + '.byte 3,32\n' + '.byte 31\n' - bytes_3_lines_3 = '.byte 213\n' + '.byte 3\n' + '.byte 32,31\n' - bytes_4_lines = '.byte 213\n' + '.byte 3\n' + '.byte 32\n' + '.byte 31\n' - mov_start_1 = 'mov x1, #111\n' - mov_start_2 = 'mov x1, 111 // should work as well\n' - mov_end_1 = 'mov x1, #222 // preferred way\n' - mov_end_2 = 'mov x1, 222\n' + bytes_1_line = ".byte 213,3,32,31\n" + bytes_2_lines_1 = ".byte 213,3,32\n" + ".byte 31\n" + bytes_2_lines_2 = ".byte 213,3\n" + ".byte 32,31\n" + bytes_2_lines_3 = ".byte 213\n" + ".byte 3,32,31\n" + bytes_3_lines_1 = ".byte 213,3\n" + ".byte 32\n" + ".byte 31\n" + bytes_3_lines_2 = ".byte 213\n" + ".byte 3,32\n" + ".byte 31\n" + bytes_3_lines_3 = ".byte 213\n" + ".byte 3\n" + ".byte 32,31\n" + bytes_4_lines = ".byte 213\n" + ".byte 3\n" + ".byte 32\n" + ".byte 31\n" + mov_start_1 = "mov x1, #111\n" + mov_start_2 = "mov x1, 111 // should work as well\n" + mov_end_1 = "mov x1, #222 // preferred way\n" + mov_end_2 = "mov x1, 222\n" prologue = ( - 'mov x12, xzr\n' - + '\tldp x9, x10, [sp, #16] // 8-byte Folded Reload\n' - + ' .p2align 6\n' + "mov x12, xzr\n" + + "\tldp x9, x10, [sp, #16] // 8-byte Folded Reload\n" + + " .p2align 6\n" ) kernel = ( - '.LBB0_28:\n' - + 'fmul v7.2d, v7.2d, v19.2d\n' - + 'stp q0, q1, [x10, #-32]\n' - + 'b.ne .LBB0_28\n' + ".LBB0_28:\n" + + "fmul v7.2d, v7.2d, v19.2d\n" + + "stp q0, q1, [x10, #-32]\n" + + "b.ne .LBB0_28\n" ) - epilogue = '.LBB0_29: // Parent Loop BB0_20 Depth=1\n' + 'bl dummy\n' - kernel_length = len(list(filter(None, kernel.split('\n')))) + epilogue = ".LBB0_29: // Parent Loop BB0_20 Depth=1\n" + "bl dummy\n" + kernel_length = len(list(filter(None, kernel.split("\n")))) bytes_variations = [ bytes_1_line, @@ -100,12 +104,12 @@ class TestMarkerUtils(unittest.TestCase): bytes_end=bytes_var_2, ): sample_parsed = self.parser_AArch.parse_file(sample_code) - sample_kernel = reduce_to_section(sample_parsed, 'AArch64') + sample_kernel = reduce_to_section(sample_parsed, "AArch64") self.assertEqual(len(sample_kernel), kernel_length) kernel_start = len( list( filter( - None, (prologue + mov_start_var + bytes_var_1).split('\n') + None, (prologue + mov_start_var + bytes_var_1).split("\n") ) ) ) @@ -116,27 +120,27 @@ class TestMarkerUtils(unittest.TestCase): def test_marker_matching_x86(self): # preparation - bytes_1_line = '.byte 100,103,144\n' - bytes_2_lines_1 = '.byte 100,103\n' + '.byte 144\n' - bytes_2_lines_2 = '.byte 100\n' + '.byte 103,144\n' + bytes_1_line = ".byte 100,103,144\n" + bytes_2_lines_1 = ".byte 100,103\n" + ".byte 144\n" + bytes_2_lines_2 = ".byte 100\n" + ".byte 103,144\n" bytes_3_lines = ( - '.byte 100 # IACA MARKER UTILITY\n' - + '.byte 103 # IACA MARKER UTILITY\n' - + '.byte 144 # IACA MARKER UTILITY\n' + ".byte 100 # IACA MARKER UTILITY\n" + + ".byte 103 # IACA MARKER UTILITY\n" + + ".byte 144 # IACA MARKER UTILITY\n" ) - mov_start_1 = 'movl $111, %ebx # IACA START\n' - mov_start_2 = 'mov $111, %ebx # IACA START\n' - mov_end_1 = 'movl $222, %ebx # IACA END\n' - mov_end_2 = 'mov $222, %ebx # IACA END\n' - prologue = 'movl -92(%rbp), %r11d\n' + 'movl $111, %ebx\n' + mov_start_1 = "movl $111, %ebx # IACA START\n" + mov_start_2 = "mov $111, %ebx # IACA START\n" + mov_end_1 = "movl $222, %ebx # IACA END\n" + mov_end_2 = "mov $222, %ebx # IACA END\n" + prologue = "movl -92(%rbp), %r11d\n" + "movl $111, %ebx\n" kernel = ( - 'vfmadd132sd (%r15,%rcx,8), %xmm5, %xmm0\n' - + 'vmovsd %xmm0, (%r14,%rcx,8)\n' - + 'cmpl %ebx, %ecx\n' - + 'jge .L8\n' + "vfmadd132sd (%r15,%rcx,8), %xmm5, %xmm0\n" + + "vmovsd %xmm0, (%r14,%rcx,8)\n" + + "cmpl %ebx, %ecx\n" + + "jge .L8\n" ) - epilogue = '.LE9:\t\t#12.2\n' 'call dummy\n' - kernel_length = len(list(filter(None, kernel.split('\n')))) + epilogue = ".LE9:\t\t#12.2\n" "call dummy\n" + kernel_length = len(list(filter(None, kernel.split("\n")))) bytes_variations = [bytes_1_line, bytes_2_lines_1, bytes_2_lines_2, bytes_3_lines] mov_start_variations = [mov_start_1, mov_start_2] @@ -162,12 +166,12 @@ class TestMarkerUtils(unittest.TestCase): bytes_end=bytes_var_2, ): sample_parsed = self.parser_x86.parse_file(sample_code) - sample_kernel = reduce_to_section(sample_parsed, 'x86') + sample_kernel = reduce_to_section(sample_parsed, "x86") self.assertEqual(len(sample_kernel), kernel_length) kernel_start = len( list( filter( - None, (prologue + mov_start_var + bytes_var_1).split('\n') + None, (prologue + mov_start_var + bytes_var_1).split("\n") ) ) ) @@ -177,171 +181,250 @@ class TestMarkerUtils(unittest.TestCase): self.assertEqual(sample_kernel, parsed_kernel) def test_marker_special_cases_AArch(self): - bytes_line = '.byte 213,3,32,31\n' - start_marker = 'mov x1, #111\n' + bytes_line - end_marker = 'mov x1, #222\n' + bytes_line - prologue = ( - 'dup v0.2d, x14\n' - 'neg x9, x9\n' - '.p2align 6\n') + bytes_line = ".byte 213,3,32,31\n" + start_marker = "mov x1, #111\n" + bytes_line + end_marker = "mov x1, #222\n" + bytes_line + prologue = "dup v0.2d, x14\n" "neg x9, x9\n" ".p2align 6\n" kernel = ( - '.LBB0_28:\n' - + 'fmul v7.2d, v7.2d, v19.2d\n' - + 'stp q0, q1, [x10, #-32]\n' - + 'b.ne .LBB0_28\n') - epilogue = ( - '.LBB0_29: // Parent Loop BB0_20 Depth=1\n' - 'bl dummy\n') + ".LBB0_28:\n" + + "fmul v7.2d, v7.2d, v19.2d\n" + + "stp q0, q1, [x10, #-32]\n" + + "b.ne .LBB0_28\n" + ) + epilogue = ".LBB0_29: // Parent Loop BB0_20 Depth=1\n" "bl dummy\n" samples = [ # (test name, # ignored prologue, section to be extraced, ignored epilogue) - ("markers", - prologue + start_marker, kernel, end_marker + epilogue), - ("marker at file start", - start_marker, kernel, end_marker + epilogue), - ("no start marker", - '', prologue + kernel, end_marker + epilogue), - ("marker at file end", - prologue + start_marker, kernel, end_marker), - ("no end marker", - prologue + start_marker, kernel + epilogue, ''), - ("empty kernel", - prologue + start_marker, '', end_marker + epilogue), + ("markers", prologue + start_marker, kernel, end_marker + epilogue), + ("marker at file start", start_marker, kernel, end_marker + epilogue), + ("no start marker", "", prologue + kernel, end_marker + epilogue), + ("marker at file end", prologue + start_marker, kernel, end_marker), + ("no end marker", prologue + start_marker, kernel + epilogue, ""), + ("empty kernel", prologue + start_marker, "", end_marker + epilogue), ] for test_name, pro, kernel, epi in samples: code = pro + kernel + epi parsed = self.parser_AArch.parse_file(code) - test_kernel = reduce_to_section(parsed, 'AArch64') + test_kernel = reduce_to_section(parsed, "AArch64") if kernel: - kernel_length = len(kernel.strip().split('\n')) + kernel_length = len(kernel.strip().split("\n")) else: kernel_length = 0 self.assertEqual( - len(test_kernel), kernel_length, - msg="Invalid exctracted kernel length on {!r} sample".format(test_name)) + len(test_kernel), + kernel_length, + msg="Invalid exctracted kernel length on {!r} sample".format(test_name), + ) if pro: - kernel_start = len((pro).strip().split('\n')) + kernel_start = len((pro).strip().split("\n")) else: kernel_start = 0 parsed_kernel = self.parser_AArch.parse_file(kernel, start_line=kernel_start) self.assertEqual( - test_kernel, parsed_kernel, - msg="Invalid exctracted kernel on {!r}".format(test_name)) + test_kernel, + parsed_kernel, + msg="Invalid exctracted kernel on {!r}".format(test_name), + ) def test_marker_special_cases_x86(self): - bytes_line = ( - '.byte 100\n' - '.byte 103\n' - '.byte 144\n') - start_marker = 'movl $111, %ebx\n' + bytes_line - end_marker = 'movl $222, %ebx\n' + bytes_line - prologue = ( - 'movl -88(%rbp), %r10d\n' - 'xorl %r11d, %r11d\n' - '.p2align 4,,10\n') + bytes_line = ".byte 100\n" ".byte 103\n" ".byte 144\n" + start_marker = "movl $111, %ebx\n" + bytes_line + end_marker = "movl $222, %ebx\n" + bytes_line + prologue = "movl -88(%rbp), %r10d\n" "xorl %r11d, %r11d\n" ".p2align 4,,10\n" kernel = ( - '.L3: #L3\n' - 'vmovsd .LC1(%rip), %xmm0\n' - 'vmovsd %xmm0, (%r15,%rcx,8)\n' - 'cmpl %ecx, %ebx\n' - 'jle .L3\n') - epilogue = ( - 'leaq -56(%rbp), %rsi\n' - 'movl %r10d, -88(%rbp)\n' - 'call timing\n') + ".L3: #L3\n" + "vmovsd .LC1(%rip), %xmm0\n" + "vmovsd %xmm0, (%r15,%rcx,8)\n" + "cmpl %ecx, %ebx\n" + "jle .L3\n" + ) + epilogue = "leaq -56(%rbp), %rsi\n" "movl %r10d, -88(%rbp)\n" "call timing\n" samples = [ # (test name, # ignored prologue, section to be extraced, ignored epilogue) - ("markers", - prologue + start_marker, kernel, end_marker + epilogue), - ("marker at file start", - start_marker, kernel, end_marker + epilogue), - ("no start marker", - '', prologue + kernel, end_marker + epilogue), - ("marker at file end", - prologue + start_marker, kernel, end_marker), - ("no end marker", - prologue + start_marker, kernel + epilogue, ''), - ("empty kernel", - prologue + start_marker, '', end_marker + epilogue), + ("markers", prologue + start_marker, kernel, end_marker + epilogue), + ("marker at file start", start_marker, kernel, end_marker + epilogue), + ("no start marker", "", prologue + kernel, end_marker + epilogue), + ("marker at file end", prologue + start_marker, kernel, end_marker), + ("no end marker", prologue + start_marker, kernel + epilogue, ""), + ("empty kernel", prologue + start_marker, "", end_marker + epilogue), ] for test_name, pro, kernel, epi in samples: code = pro + kernel + epi parsed = self.parser_x86.parse_file(code) - test_kernel = reduce_to_section(parsed, 'x86') + test_kernel = reduce_to_section(parsed, "x86") if kernel: - kernel_length = len(kernel.strip().split('\n')) + kernel_length = len(kernel.strip().split("\n")) else: kernel_length = 0 self.assertEqual( - len(test_kernel), kernel_length, - msg="Invalid exctracted kernel length on {!r} sample".format(test_name)) + len(test_kernel), + kernel_length, + msg="Invalid exctracted kernel length on {!r} sample".format(test_name), + ) if pro: - kernel_start = len((pro).strip().split('\n')) + kernel_start = len((pro).strip().split("\n")) else: kernel_start = 0 parsed_kernel = self.parser_x86.parse_file(kernel, start_line=kernel_start) self.assertEqual( - test_kernel, parsed_kernel, - msg="Invalid exctracted kernel on {!r}".format(test_name)) + test_kernel, + parsed_kernel, + msg="Invalid exctracted kernel on {!r}".format(test_name), + ) def test_find_jump_labels(self): - self.assertEqual(find_jump_labels(self.parsed_x86), - OrderedDict([('.LFB24', 10), ('.L4', 65), ('.L3', 79), ('.L2', 102), - ('.L13', 111), ('.L12', 120), ('.L6', 132), ('.L10', 145), - ('.L9', 161), ('.L8', 183), ('.L15', 252), ('.L26', 256), - ('.L14', 259), ('.LFB25', 277), ('.L28', 289)])) + self.assertEqual( + find_jump_labels(self.parsed_x86), + OrderedDict( + [ + (".LFB24", 10), + (".L4", 65), + (".L3", 79), + (".L2", 102), + (".L13", 111), + (".L12", 120), + (".L6", 132), + (".L10", 145), + (".L9", 161), + (".L8", 183), + (".L15", 252), + (".L26", 256), + (".L14", 259), + (".LFB25", 277), + (".L28", 289), + ] + ), + ) self.assertEqual( find_jump_labels(self.parsed_AArch), - OrderedDict([('triad', 18), ('.LBB0_3', 71), ('.LBB0_4', 76), ('.LBB0_5', 84), - ('.LBB0_7', 92), ('.LBB0_8', 95), ('.LBB0_9', 106), ('.LBB0_11', 118), - ('.LBB0_12', 133), ('.LBB0_14', 177), ('.LBB0_15', 190), - ('.LBB0_16', 205), ('.LBB0_17', 208), ('.LBB0_18', 221), - ('.LBB0_19', 228), ('.LBB0_20', 260), ('.LBB0_22', 272), - ('.LBB0_24', 283), ('.LBB0_26', 290), ('.LBB0_28', 298), - ('.LBB0_29', 306), ('.LBB0_31', 448), ('.LBB0_32', 458), - ('.LBB0_33', 480), ('.LBB0_34', 484), ('.LBB0_35', 493), - ('.LBB0_36', 504), ('.LBB0_37', 508), ('.LBB0_38', 518), - ('main', 574)])) + OrderedDict( + [ + ("triad", 18), + (".LBB0_3", 71), + (".LBB0_4", 76), + (".LBB0_5", 84), + (".LBB0_7", 92), + (".LBB0_8", 95), + (".LBB0_9", 106), + (".LBB0_11", 118), + (".LBB0_12", 133), + (".LBB0_14", 177), + (".LBB0_15", 190), + (".LBB0_16", 205), + (".LBB0_17", 208), + (".LBB0_18", 221), + (".LBB0_19", 228), + (".LBB0_20", 260), + (".LBB0_22", 272), + (".LBB0_24", 283), + (".LBB0_26", 290), + (".LBB0_28", 298), + (".LBB0_29", 306), + (".LBB0_31", 448), + (".LBB0_32", 458), + (".LBB0_33", 480), + (".LBB0_34", 484), + (".LBB0_35", 493), + (".LBB0_36", 504), + (".LBB0_37", 508), + (".LBB0_38", 518), + ("main", 574), + ] + ), + ) def test_find_basic_blocks(self): self.assertEqual( - [(k, v[0]['line_number'], v[-1]['line_number']) - for k, v in find_basic_blocks(self.parsed_x86).items()], - [('.LFB24', 11, 56), ('.L4', 66, 74), ('.L3', 80, 89), ('.L2', 103, 112), - ('.L13', 112, 121), ('.L12', 121, 125), ('.L6', 133, 135), ('.L10', 146, 154), - ('.L9', 162, 170), ('.L8', 184, 187), ('.L15', 253, 256), ('.L26', 257, 259), - ('.L14', 260, 262), ('.LFB25', 278, 290), ('.L28', 290, 300)]) + [ + (k, v[0]["line_number"], v[-1]["line_number"]) + for k, v in find_basic_blocks(self.parsed_x86).items() + ], + [ + (".LFB24", 11, 56), + (".L4", 66, 74), + (".L3", 80, 89), + (".L2", 103, 112), + (".L13", 112, 121), + (".L12", 121, 125), + (".L6", 133, 135), + (".L10", 146, 154), + (".L9", 162, 170), + (".L8", 184, 187), + (".L15", 253, 256), + (".L26", 257, 259), + (".L14", 260, 262), + (".LFB25", 278, 290), + (".L28", 290, 300), + ], + ) self.assertEqual( - [(k, v[0]['line_number'], v[-1]['line_number']) - for k, v in find_basic_blocks(self.parsed_AArch).items()], - [('triad', 19, 64), ('.LBB0_3', 72, 77), ('.LBB0_4', 77, 83), ('.LBB0_5', 85, 89), - ('.LBB0_7', 93, 95), ('.LBB0_8', 96, 105), ('.LBB0_9', 107, 114), - ('.LBB0_11', 119, 134), ('.LBB0_12', 134, 173), ('.LBB0_14', 178, 191), - ('.LBB0_15', 191, 205), ('.LBB0_16', 206, 208), ('.LBB0_17', 209, 222), - ('.LBB0_18', 222, 228), ('.LBB0_19', 229, 261), ('.LBB0_20', 261, 269), - ('.LBB0_22', 273, 280), ('.LBB0_24', 284, 286), ('.LBB0_26', 291, 293), - ('.LBB0_28', 299, 307), ('.LBB0_29', 307, 444), ('.LBB0_31', 449, 459), - ('.LBB0_32', 459, 480), ('.LBB0_33', 481, 484), ('.LBB0_34', 485, 494), - ('.LBB0_35', 494, 504), ('.LBB0_36', 505, 508), ('.LBB0_37', 509, 518), - ('.LBB0_38', 519, 568), ('main', 575, 590)]) + [ + (k, v[0]["line_number"], v[-1]["line_number"]) + for k, v in find_basic_blocks(self.parsed_AArch).items() + ], + [ + ("triad", 19, 64), + (".LBB0_3", 72, 77), + (".LBB0_4", 77, 83), + (".LBB0_5", 85, 89), + (".LBB0_7", 93, 95), + (".LBB0_8", 96, 105), + (".LBB0_9", 107, 114), + (".LBB0_11", 119, 134), + (".LBB0_12", 134, 173), + (".LBB0_14", 178, 191), + (".LBB0_15", 191, 205), + (".LBB0_16", 206, 208), + (".LBB0_17", 209, 222), + (".LBB0_18", 222, 228), + (".LBB0_19", 229, 261), + (".LBB0_20", 261, 269), + (".LBB0_22", 273, 280), + (".LBB0_24", 284, 286), + (".LBB0_26", 291, 293), + (".LBB0_28", 299, 307), + (".LBB0_29", 307, 444), + (".LBB0_31", 449, 459), + (".LBB0_32", 459, 480), + (".LBB0_33", 481, 484), + (".LBB0_34", 485, 494), + (".LBB0_35", 494, 504), + (".LBB0_36", 505, 508), + (".LBB0_37", 509, 518), + (".LBB0_38", 519, 568), + ("main", 575, 590), + ], + ) def test_find_basic_loop_body(self): self.assertEqual( - [(k, v[0]['line_number'], v[-1]['line_number']) - for k, v in find_basic_loop_bodies(self.parsed_x86).items()], - [('.L4', 66, 74), ('.L10', 146, 154), ('.L28', 290, 300)]) + [ + (k, v[0]["line_number"], v[-1]["line_number"]) + for k, v in find_basic_loop_bodies(self.parsed_x86).items() + ], + [(".L4", 66, 74), (".L10", 146, 154), (".L28", 290, 300)], + ) self.assertEqual( - [(k, v[0]['line_number'], v[-1]['line_number']) - for k, v in find_basic_loop_bodies(self.parsed_AArch).items()], - [('.LBB0_12', 134, 173), ('.LBB0_15', 191, 205), ('.LBB0_18', 222, 228), - ('.LBB0_29', 307, 444), ('.LBB0_32', 459, 480), ('.LBB0_35', 494, 504)]) + [ + (k, v[0]["line_number"], v[-1]["line_number"]) + for k, v in find_basic_loop_bodies(self.parsed_AArch).items() + ], + [ + (".LBB0_12", 134, 173), + (".LBB0_15", 191, 205), + (".LBB0_18", 222, 228), + (".LBB0_29", 307, 444), + (".LBB0_32", 459, 480), + (".LBB0_35", 494, 504), + ], + ) ################## # Helper functions @@ -350,11 +433,11 @@ class TestMarkerUtils(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestMarkerUtils) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_parser_AArch64.py b/tests/test_parser_AArch64.py index fa8d20f..aace181 100755 --- a/tests/test_parser_AArch64.py +++ b/tests/test_parser_AArch64.py @@ -15,7 +15,7 @@ class TestParserAArch64(unittest.TestCase): @classmethod def setUpClass(self): self.parser = ParserAArch64() - with open(self._find_file('triad_arm_iaca.s')) as f: + with open(self._find_file("triad_arm_iaca.s")) as f: self.triad_code = f.read() ################## @@ -23,63 +23,61 @@ class TestParserAArch64(unittest.TestCase): ################## def test_comment_parser(self): - self.assertEqual(self._get_comment(self.parser, '// some comments'), 'some comments') + self.assertEqual(self._get_comment(self.parser, "// some comments"), "some comments") + self.assertEqual(self._get_comment(self.parser, "\t\t//AA BB CC \t end \t"), "AA BB CC end") self.assertEqual( - self._get_comment(self.parser, '\t\t//AA BB CC \t end \t'), 'AA BB CC end' - ) - self.assertEqual( - self._get_comment(self.parser, '\t//// comment //// comment'), - '// comment //// comment', + self._get_comment(self.parser, "\t//// comment //// comment"), + "// comment //// comment", ) def test_label_parser(self): - self.assertEqual(self._get_label(self.parser, 'main:').name, 'main') - self.assertEqual(self._get_label(self.parser, '..B1.10:').name, '..B1.10') - self.assertEqual(self._get_label(self.parser, '.2.3_2_pack.3:').name, '.2.3_2_pack.3') - self.assertEqual(self._get_label(self.parser, '.L1:\t\t\t//label1').name, '.L1') + self.assertEqual(self._get_label(self.parser, "main:").name, "main") + self.assertEqual(self._get_label(self.parser, "..B1.10:").name, "..B1.10") + self.assertEqual(self._get_label(self.parser, ".2.3_2_pack.3:").name, ".2.3_2_pack.3") + self.assertEqual(self._get_label(self.parser, ".L1:\t\t\t//label1").name, ".L1") self.assertEqual( - ' '.join(self._get_label(self.parser, '.L1:\t\t\t//label1').comment), 'label1' + " ".join(self._get_label(self.parser, ".L1:\t\t\t//label1").comment), "label1" ) with self.assertRaises(ParseException): - self._get_label(self.parser, '\t.cfi_startproc') + self._get_label(self.parser, "\t.cfi_startproc") def test_directive_parser(self): - self.assertEqual(self._get_directive(self.parser, '\t.text').name, 'text') - self.assertEqual(len(self._get_directive(self.parser, '\t.text').parameters), 0) - self.assertEqual(self._get_directive(self.parser, '\t.align\t16,0x90').name, 'align') - self.assertEqual(len(self._get_directive(self.parser, '\t.align\t16,0x90').parameters), 2) + self.assertEqual(self._get_directive(self.parser, "\t.text").name, "text") + self.assertEqual(len(self._get_directive(self.parser, "\t.text").parameters), 0) + self.assertEqual(self._get_directive(self.parser, "\t.align\t16,0x90").name, "align") + self.assertEqual(len(self._get_directive(self.parser, "\t.align\t16,0x90").parameters), 2) self.assertEqual( - self._get_directive(self.parser, '\t.align\t16,0x90').parameters[1], '0x90' + self._get_directive(self.parser, "\t.align\t16,0x90").parameters[1], "0x90" ) self.assertEqual( - self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[ - 'name' + self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[ + "name" ], - 'byte', + "byte", ) self.assertEqual( - self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[ - 'parameters' + self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[ + "parameters" ][2], - '144', + "144", ) self.assertEqual( - ' '.join( - self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[ - 'comment' + " ".join( + self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[ + "comment" ] ), - 'IACA START', + "IACA START", ) def test_parse_instruction(self): - instr1 = '\t\tvcvt.F32.S32 w1, w2\t\t\t//12.27' - instr2 = 'b.lo ..B1.4 \t' - instr3 = ' mov x2,#0x222 //NOT IACA END' - instr4 = 'str x28, [sp, x1, lsl #4] //12.9' - instr5 = 'ldr x0, [x0, #:got_lo12:q2c]' - instr6 = 'adrp x0, :got:visited' - instr7 = 'fadd v17.2d, v16.2d, v1.2d' + instr1 = "\t\tvcvt.F32.S32 w1, w2\t\t\t//12.27" + instr2 = "b.lo ..B1.4 \t" + instr3 = " mov x2,#0x222 //NOT IACA END" + instr4 = "str x28, [sp, x1, lsl #4] //12.9" + instr5 = "ldr x0, [x0, #:got_lo12:q2c]" + instr6 = "adrp x0, :got:visited" + instr7 = "fadd v17.2d, v16.2d, v1.2d" parsed_1 = self.parser.parse_instruction(instr1) parsed_2 = self.parser.parse_instruction(instr2) @@ -89,198 +87,196 @@ class TestParserAArch64(unittest.TestCase): parsed_6 = self.parser.parse_instruction(instr6) parsed_7 = self.parser.parse_instruction(instr7) - self.assertEqual(parsed_1.instruction, 'vcvt.F32.S32') - self.assertEqual(parsed_1.operands[0].register.name, '1') - self.assertEqual(parsed_1.operands[0].register.prefix, 'w') - self.assertEqual(parsed_1.operands[1].register.name, '2') - self.assertEqual(parsed_1.operands[1].register.prefix, 'w') - self.assertEqual(parsed_1.comment, '12.27') + self.assertEqual(parsed_1.instruction, "vcvt.F32.S32") + self.assertEqual(parsed_1.operands[0].register.name, "1") + self.assertEqual(parsed_1.operands[0].register.prefix, "w") + self.assertEqual(parsed_1.operands[1].register.name, "2") + self.assertEqual(parsed_1.operands[1].register.prefix, "w") + self.assertEqual(parsed_1.comment, "12.27") - self.assertEqual(parsed_2.instruction, 'b.lo') - self.assertEqual(parsed_2.operands[0].identifier.name, '..B1.4') + self.assertEqual(parsed_2.instruction, "b.lo") + self.assertEqual(parsed_2.operands[0].identifier.name, "..B1.4") self.assertEqual(len(parsed_2.operands), 1) self.assertIsNone(parsed_2.comment) - self.assertEqual(parsed_3.instruction, 'mov') - self.assertEqual(parsed_3.operands[0].register.name, '2') - self.assertEqual(parsed_3.operands[0].register.prefix, 'x') - self.assertEqual(parsed_3.operands[1].immediate.value, '0x222') - self.assertEqual(parsed_3.comment, 'NOT IACA END') + self.assertEqual(parsed_3.instruction, "mov") + self.assertEqual(parsed_3.operands[0].register.name, "2") + self.assertEqual(parsed_3.operands[0].register.prefix, "x") + self.assertEqual(parsed_3.operands[1].immediate.value, "0x222") + self.assertEqual(parsed_3.comment, "NOT IACA END") - self.assertEqual(parsed_4.instruction, 'str') + self.assertEqual(parsed_4.instruction, "str") self.assertIsNone(parsed_4.operands[1].memory.offset) - self.assertEqual(parsed_4.operands[1].memory.base.name, 'sp') - self.assertEqual(parsed_4.operands[1].memory.base.prefix, 'x') - self.assertEqual(parsed_4.operands[1].memory.index.name, '1') - self.assertEqual(parsed_4.operands[1].memory.index.prefix, 'x') + self.assertEqual(parsed_4.operands[1].memory.base.name, "sp") + self.assertEqual(parsed_4.operands[1].memory.base.prefix, "x") + self.assertEqual(parsed_4.operands[1].memory.index.name, "1") + self.assertEqual(parsed_4.operands[1].memory.index.prefix, "x") self.assertEqual(parsed_4.operands[1].memory.scale, 16) - self.assertEqual(parsed_4.operands[0].register.name, '28') - self.assertEqual(parsed_4.operands[0].register.prefix, 'x') - self.assertEqual(parsed_4.comment, '12.9') + self.assertEqual(parsed_4.operands[0].register.name, "28") + self.assertEqual(parsed_4.operands[0].register.prefix, "x") + self.assertEqual(parsed_4.comment, "12.9") - self.assertEqual(parsed_5.instruction, 'ldr') - self.assertEqual(parsed_5.operands[0].register.name, '0') - self.assertEqual(parsed_5.operands[0].register.prefix, 'x') - self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, 'q2c') - self.assertEqual(parsed_5.operands[1].memory.offset.identifier.relocation, - ':got_lo12:') - self.assertEqual(parsed_5.operands[1].memory.base.name, '0') - self.assertEqual(parsed_5.operands[1].memory.base.prefix, 'x') + self.assertEqual(parsed_5.instruction, "ldr") + self.assertEqual(parsed_5.operands[0].register.name, "0") + self.assertEqual(parsed_5.operands[0].register.prefix, "x") + self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, "q2c") + self.assertEqual(parsed_5.operands[1].memory.offset.identifier.relocation, ":got_lo12:") + self.assertEqual(parsed_5.operands[1].memory.base.name, "0") + self.assertEqual(parsed_5.operands[1].memory.base.prefix, "x") self.assertIsNone(parsed_5.operands[1].memory.index) self.assertEqual(parsed_5.operands[1].memory.scale, 1) - self.assertEqual(parsed_6.instruction, 'adrp') - self.assertEqual(parsed_6.operands[0].register.name, '0') - self.assertEqual(parsed_6.operands[0].register.prefix, 'x') - self.assertEqual(parsed_6.operands[1].identifier.relocation, ':got:') - self.assertEqual(parsed_6.operands[1].identifier.name, 'visited') + self.assertEqual(parsed_6.instruction, "adrp") + self.assertEqual(parsed_6.operands[0].register.name, "0") + self.assertEqual(parsed_6.operands[0].register.prefix, "x") + self.assertEqual(parsed_6.operands[1].identifier.relocation, ":got:") + self.assertEqual(parsed_6.operands[1].identifier.name, "visited") - self.assertEqual(parsed_7.instruction, 'fadd') - self.assertEqual(parsed_7.operands[0].register.name, '17') - self.assertEqual(parsed_7.operands[0].register.prefix, 'v') - self.assertEqual(parsed_7.operands[0].register.lanes, '2') - self.assertEqual(parsed_7.operands[0].register.shape, 'd') - self.assertEqual(self.parser.get_full_reg_name(parsed_7.operands[2].register), - 'v1.2d') + self.assertEqual(parsed_7.instruction, "fadd") + self.assertEqual(parsed_7.operands[0].register.name, "17") + self.assertEqual(parsed_7.operands[0].register.prefix, "v") + self.assertEqual(parsed_7.operands[0].register.lanes, "2") + self.assertEqual(parsed_7.operands[0].register.shape, "d") + self.assertEqual(self.parser.get_full_reg_name(parsed_7.operands[2].register), "v1.2d") def test_parse_line(self): - line_comment = '// -- Begin main' - line_label = '.LBB0_1: // =>This Inner Loop Header: Depth=1' - line_directive = '.cfi_def_cfa w29, -16' - line_instruction = 'ldr s0, [x11, w10, sxtw #2] // = <<2' - line_prefetch = 'prfm pldl1keep, [x26, #2048] //HPL' - line_preindexed = 'stp x29, x30, [sp, #-16]!' - line_postindexed = 'ldp q2, q3, [x11], #64' - line_5_operands = 'fcmla z26.d, p0/m, z29.d, z21.d, #90' + line_comment = "// -- Begin main" + line_label = ".LBB0_1: // =>This Inner Loop Header: Depth=1" + line_directive = ".cfi_def_cfa w29, -16" + line_instruction = "ldr s0, [x11, w10, sxtw #2] // = <<2" + line_prefetch = "prfm pldl1keep, [x26, #2048] //HPL" + line_preindexed = "stp x29, x30, [sp, #-16]!" + line_postindexed = "ldp q2, q3, [x11], #64" + line_5_operands = "fcmla z26.d, p0/m, z29.d, z21.d, #90" instruction_form_1 = { - 'instruction': None, - 'operands': [], - 'directive': None, - 'comment': '-- Begin main', - 'label': None, - 'line': '// -- Begin main', - 'line_number': 1, + "instruction": None, + "operands": [], + "directive": None, + "comment": "-- Begin main", + "label": None, + "line": "// -- Begin main", + "line_number": 1, } instruction_form_2 = { - 'instruction': None, - 'operands': [], - 'directive': None, - 'comment': '=>This Inner Loop Header: Depth=1', - 'label': '.LBB0_1', - 'line': '.LBB0_1: // =>This Inner Loop Header: Depth=1', - 'line_number': 2, + "instruction": None, + "operands": [], + "directive": None, + "comment": "=>This Inner Loop Header: Depth=1", + "label": ".LBB0_1", + "line": ".LBB0_1: // =>This Inner Loop Header: Depth=1", + "line_number": 2, } instruction_form_3 = { - 'instruction': None, - 'operands': [], - 'directive': {'name': 'cfi_def_cfa', 'parameters': ['w29', '-16']}, - 'comment': None, - 'label': None, - 'line': '.cfi_def_cfa w29, -16', - 'line_number': 3, + "instruction": None, + "operands": [], + "directive": {"name": "cfi_def_cfa", "parameters": ["w29", "-16"]}, + "comment": None, + "label": None, + "line": ".cfi_def_cfa w29, -16", + "line_number": 3, } instruction_form_4 = { - 'instruction': 'ldr', - 'operands': [ - {'register': {'prefix': 's', 'name': '0'}}, + "instruction": "ldr", + "operands": [ + {"register": {"prefix": "s", "name": "0"}}, { - 'memory': { - 'offset': None, - 'base': {'prefix': 'x', 'name': '11'}, - 'index': { - 'prefix': 'w', - 'name': '10', - 'shift_op': 'sxtw', - 'immediate': {'value': '2'}, - 'shift': [{'value': '2'}], + "memory": { + "offset": None, + "base": {"prefix": "x", "name": "11"}, + "index": { + "prefix": "w", + "name": "10", + "shift_op": "sxtw", + "immediate": {"value": "2"}, + "shift": [{"value": "2"}], }, - 'scale': 4, + "scale": 4, } }, ], - 'directive': None, - 'comment': '= <<2', - 'label': None, - 'line': 'ldr s0, [x11, w10, sxtw #2] // = <<2', - 'line_number': 4, + "directive": None, + "comment": "= <<2", + "label": None, + "line": "ldr s0, [x11, w10, sxtw #2] // = <<2", + "line_number": 4, } instruction_form_5 = { - 'instruction': 'prfm', - 'operands': [ - {'prfop': {'type': ['PLD'], 'target': ['L1'], 'policy': ['KEEP']}}, + "instruction": "prfm", + "operands": [ + {"prfop": {"type": ["PLD"], "target": ["L1"], "policy": ["KEEP"]}}, { - 'memory': { - 'offset': {'value': '2048'}, - 'base': {'prefix': 'x', 'name': '26'}, - 'index': None, - 'scale': 1, + "memory": { + "offset": {"value": "2048"}, + "base": {"prefix": "x", "name": "26"}, + "index": None, + "scale": 1, } }, ], - 'directive': None, - 'comment': 'HPL', - 'label': None, - 'line': 'prfm pldl1keep, [x26, #2048] //HPL', - 'line_number': 5, + "directive": None, + "comment": "HPL", + "label": None, + "line": "prfm pldl1keep, [x26, #2048] //HPL", + "line_number": 5, } instruction_form_6 = { - 'instruction': 'stp', - 'operands': [ - {'register': {'prefix': 'x', 'name': '29'}}, - {'register': {'prefix': 'x', 'name': '30'}}, + "instruction": "stp", + "operands": [ + {"register": {"prefix": "x", "name": "29"}}, + {"register": {"prefix": "x", "name": "30"}}, { - 'memory': { - 'offset': {'value': '-16'}, - 'base': {'name': 'sp', 'prefix': 'x'}, - 'index': None, - 'scale': 1, - 'pre_indexed': True, + "memory": { + "offset": {"value": "-16"}, + "base": {"name": "sp", "prefix": "x"}, + "index": None, + "scale": 1, + "pre_indexed": True, } }, ], - 'directive': None, - 'comment': None, - 'label': None, - 'line': 'stp x29, x30, [sp, #-16]!', - 'line_number': 6, + "directive": None, + "comment": None, + "label": None, + "line": "stp x29, x30, [sp, #-16]!", + "line_number": 6, } instruction_form_7 = { - 'instruction': 'ldp', - 'operands': [ - {'register': {'prefix': 'q', 'name': '2'}}, - {'register': {'prefix': 'q', 'name': '3'}}, + "instruction": "ldp", + "operands": [ + {"register": {"prefix": "q", "name": "2"}}, + {"register": {"prefix": "q", "name": "3"}}, { - 'memory': { - 'offset': None, - 'base': {'prefix': 'x', 'name': '11'}, - 'index': None, - 'scale': 1, - 'post_indexed': {'value': '64'}, + "memory": { + "offset": None, + "base": {"prefix": "x", "name": "11"}, + "index": None, + "scale": 1, + "post_indexed": {"value": "64"}, } }, ], - 'directive': None, - 'comment': None, - 'label': None, - 'line': 'ldp q2, q3, [x11], #64', - 'line_number': 7, + "directive": None, + "comment": None, + "label": None, + "line": "ldp q2, q3, [x11], #64", + "line_number": 7, } instruction_form_8 = { - 'instruction': 'fcmla', - 'operands': [ - {'register': {'prefix': 'z', 'name': '26', 'shape': 'd'}}, - {'register': {'prefix': 'p', 'name': '0', 'predication': 'm'}}, - {'register': {'prefix': 'z', 'name': '29', 'shape': 'd'}}, - {'register': {'prefix': 'z', 'name': '21', 'shape': 'd'}}, - {'immediate': {'value': '90'}}, + "instruction": "fcmla", + "operands": [ + {"register": {"prefix": "z", "name": "26", "shape": "d"}}, + {"register": {"prefix": "p", "name": "0", "predication": "m"}}, + {"register": {"prefix": "z", "name": "29", "shape": "d"}}, + {"register": {"prefix": "z", "name": "21", "shape": "d"}}, + {"immediate": {"value": "90"}}, ], - 'directive': None, - 'comment': None, - 'label': None, - 'line': 'fcmla z26.d, p0/m, z29.d, z21.d, #90', - 'line_number': 8, + "directive": None, + "comment": None, + "label": None, + "line": "fcmla z26.d, p0/m, z29.d, z21.d, #90", + "line_number": 8, } parsed_1 = self.parser.parse_line(line_comment, 1) @@ -307,15 +303,15 @@ class TestParserAArch64(unittest.TestCase): self.assertEqual(len(parsed), 645) def test_normalize_imd(self): - imd_decimal_1 = {'value': '79'} - imd_hex_1 = {'value': '0x4f'} - imd_decimal_2 = {'value': '8'} - imd_hex_2 = {'value': '0x8'} - imd_float_11 = {'float': {'mantissa': '0.79', 'e_sign': '+', 'exponent': '2'}} - imd_float_12 = {'float': {'mantissa': '790.0', 'e_sign': '-', 'exponent': '1'}} - imd_double_11 = {'double': {'mantissa': '0.79', 'e_sign': '+', 'exponent': '2'}} - imd_double_12 = {'double': {'mantissa': '790.0', 'e_sign': '-', 'exponent': '1'}} - identifier = {'identifier': {'name': '..B1.4'}} + imd_decimal_1 = {"value": "79"} + imd_hex_1 = {"value": "0x4f"} + imd_decimal_2 = {"value": "8"} + imd_hex_2 = {"value": "0x8"} + imd_float_11 = {"float": {"mantissa": "0.79", "e_sign": "+", "exponent": "2"}} + imd_float_12 = {"float": {"mantissa": "790.0", "e_sign": "-", "exponent": "1"}} + imd_double_11 = {"double": {"mantissa": "0.79", "e_sign": "+", "exponent": "2"}} + imd_double_12 = {"double": {"mantissa": "790.0", "e_sign": "-", "exponent": "1"}} + identifier = {"identifier": {"name": "..B1.4"}} value1 = self.parser.normalize_imd(imd_decimal_1) self.assertEqual(value1, self.parser.normalize_imd(imd_hex_1)) @@ -329,27 +325,28 @@ class TestParserAArch64(unittest.TestCase): self.assertEqual(self.parser.normalize_imd(identifier), identifier) def test_multiple_regs(self): - instr_range = 'PUSH {x5-x7}' - reg_range = AttrDict({ - 'register': { - 'range': [ - {'prefix': 'x', 'name': '5'}, - {'prefix': 'x', 'name': '7'} - ], - 'index': None + instr_range = "PUSH {x5-x7}" + reg_range = AttrDict( + { + "register": { + "range": [{"prefix": "x", "name": "5"}, {"prefix": "x", "name": "7"}], + "index": None, + } } - }) - instr_list = 'POP {x5, x7, x9}' - reg_list = AttrDict({ - 'register': { - 'list': [ - {'prefix': 'x', 'name': '5'}, - {'prefix': 'x', 'name': '7'}, - {'prefix': 'x', 'name': '9'} - ], - 'index': None + ) + instr_list = "POP {x5, x7, x9}" + reg_list = AttrDict( + { + "register": { + "list": [ + {"prefix": "x", "name": "5"}, + {"prefix": "x", "name": "7"}, + {"prefix": "x", "name": "9"}, + ], + "index": None, + } } - }) + ) prange = self.parser.parse_line(instr_range) plist = self.parser.parse_line(instr_list) @@ -357,22 +354,22 @@ class TestParserAArch64(unittest.TestCase): self.assertEqual(plist.operands[0], reg_list) def test_reg_dependency(self): - reg_1_1 = AttrDict({'prefix': 'b', 'name': '1'}) - reg_1_2 = AttrDict({'prefix': 'h', 'name': '1'}) - reg_1_3 = AttrDict({'prefix': 's', 'name': '1'}) - reg_1_4 = AttrDict({'prefix': 'd', 'name': '1'}) - reg_1_4 = AttrDict({'prefix': 'q', 'name': '1'}) - reg_2_1 = AttrDict({'prefix': 'w', 'name': '2'}) - reg_2_2 = AttrDict({'prefix': 'x', 'name': '2'}) - reg_v1_1 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '16', 'shape': 'b'}) - reg_v1_2 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '8', 'shape': 'h'}) - reg_v1_3 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '4', 'shape': 's'}) - reg_v1_4 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '2', 'shape': 'd'}) + reg_1_1 = AttrDict({"prefix": "b", "name": "1"}) + reg_1_2 = AttrDict({"prefix": "h", "name": "1"}) + reg_1_3 = AttrDict({"prefix": "s", "name": "1"}) + reg_1_4 = AttrDict({"prefix": "d", "name": "1"}) + reg_1_4 = AttrDict({"prefix": "q", "name": "1"}) + reg_2_1 = AttrDict({"prefix": "w", "name": "2"}) + reg_2_2 = AttrDict({"prefix": "x", "name": "2"}) + reg_v1_1 = AttrDict({"prefix": "v", "name": "11", "lanes": "16", "shape": "b"}) + reg_v1_2 = AttrDict({"prefix": "v", "name": "11", "lanes": "8", "shape": "h"}) + reg_v1_3 = AttrDict({"prefix": "v", "name": "11", "lanes": "4", "shape": "s"}) + reg_v1_4 = AttrDict({"prefix": "v", "name": "11", "lanes": "2", "shape": "d"}) - reg_b5 = AttrDict({'prefix': 'b', 'name': '5'}) - reg_q15 = AttrDict({'prefix': 'q', 'name': '15'}) - reg_v10 = AttrDict({'prefix': 'v', 'name': '10', 'lanes': '2', 'shape': 's'}) - reg_v20 = AttrDict({'prefix': 'v', 'name': '20', 'lanes': '2', 'shape': 'd'}) + reg_b5 = AttrDict({"prefix": "b", "name": "5"}) + reg_q15 = AttrDict({"prefix": "q", "name": "15"}) + reg_v10 = AttrDict({"prefix": "v", "name": "10", "lanes": "2", "shape": "s"}) + reg_v20 = AttrDict({"prefix": "v", "name": "20", "lanes": "2", "shape": "d"}) reg_1 = [reg_1_1, reg_1_2, reg_1_3, reg_1_4] reg_2 = [reg_2_1, reg_2_2] @@ -406,7 +403,7 @@ class TestParserAArch64(unittest.TestCase): # Helper functions ################## def _get_comment(self, parser, comment): - return ' '.join( + return " ".join( AttrDict.convert_dict( parser.process_operand(parser.comment.parseString(comment, parseAll=True).asDict()) ).comment @@ -425,11 +422,11 @@ class TestParserAArch64(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestParserAArch64) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_parser_x86att.py b/tests/test_parser_x86att.py index 253518c..c9f2e89 100755 --- a/tests/test_parser_x86att.py +++ b/tests/test_parser_x86att.py @@ -15,7 +15,7 @@ class TestParserX86ATT(unittest.TestCase): @classmethod def setUpClass(self): self.parser = ParserX86ATT() - with open(self._find_file('triad_x86_iaca.s')) as f: + with open(self._find_file("triad_x86_iaca.s")) as f: self.triad_code = f.read() ################## @@ -23,29 +23,29 @@ class TestParserX86ATT(unittest.TestCase): ################## def test_comment_parser(self): - self.assertEqual(self._get_comment(self.parser, '# some comments'), 'some comments') - self.assertEqual(self._get_comment(self.parser, '\t\t#AA BB CC \t end \t'), 'AA BB CC end') + self.assertEqual(self._get_comment(self.parser, "# some comments"), "some comments") + self.assertEqual(self._get_comment(self.parser, "\t\t#AA BB CC \t end \t"), "AA BB CC end") self.assertEqual( - self._get_comment(self.parser, '\t## comment ## comment'), '# comment ## comment' + self._get_comment(self.parser, "\t## comment ## comment"), "# comment ## comment" ) def test_label_parser(self): - self.assertEqual(self._get_label(self.parser, 'main:').name, 'main') - self.assertEqual(self._get_label(self.parser, '..B1.10:').name, '..B1.10') - self.assertEqual(self._get_label(self.parser, '.2.3_2_pack.3:').name, '.2.3_2_pack.3') - self.assertEqual(self._get_label(self.parser, '.L1:\t\t\t#label1').name, '.L1') + self.assertEqual(self._get_label(self.parser, "main:").name, "main") + self.assertEqual(self._get_label(self.parser, "..B1.10:").name, "..B1.10") + self.assertEqual(self._get_label(self.parser, ".2.3_2_pack.3:").name, ".2.3_2_pack.3") + self.assertEqual(self._get_label(self.parser, ".L1:\t\t\t#label1").name, ".L1") self.assertEqual( - ' '.join(self._get_label(self.parser, '.L1:\t\t\t#label1').comment), 'label1' + " ".join(self._get_label(self.parser, ".L1:\t\t\t#label1").comment), "label1" ) with self.assertRaises(ParseException): - self._get_label(self.parser, '\t.cfi_startproc') + self._get_label(self.parser, "\t.cfi_startproc") def test_directive_parser(self): - self.assertEqual(self._get_directive(self.parser, '\t.text').name, 'text') - self.assertEqual(len(self._get_directive(self.parser, '\t.text').parameters), 0) - self.assertEqual(self._get_directive(self.parser, '\t.align\t16,0x90').name, 'align') - self.assertEqual(len(self._get_directive(self.parser, '\t.align\t16,0x90').parameters), 2) - self.assertEqual(len(self._get_directive(self.parser, '.text').parameters), 0) + self.assertEqual(self._get_directive(self.parser, "\t.text").name, "text") + self.assertEqual(len(self._get_directive(self.parser, "\t.text").parameters), 0) + self.assertEqual(self._get_directive(self.parser, "\t.align\t16,0x90").name, "align") + self.assertEqual(len(self._get_directive(self.parser, "\t.align\t16,0x90").parameters), 2) + self.assertEqual(len(self._get_directive(self.parser, ".text").parameters), 0) self.assertEqual( len(self._get_directive(self.parser, '.file\t1 "path/to/file.c"').parameters), 2 ) @@ -54,54 +54,52 @@ class TestParserX86ATT(unittest.TestCase): '"path/to/file.c"', ) self.assertEqual( - self._get_directive(self.parser, '\t.set\tL$set$0,LECIE1-LSCIE1').parameters, - ['L$set$0', 'LECIE1-LSCIE1'], + self._get_directive(self.parser, "\t.set\tL$set$0,LECIE1-LSCIE1").parameters, + ["L$set$0", "LECIE1-LSCIE1"], ) self.assertEqual( self._get_directive( self.parser, - '\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support', + "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support", ).parameters, - ['__TEXT', '__eh_frame', 'coalesced', 'no_toc+strip_static_syms+live_support'], + ["__TEXT", "__eh_frame", "coalesced", "no_toc+strip_static_syms+live_support"], ) self.assertEqual( self._get_directive( - self.parser, '\t.section\t__TEXT,__literal16,16byte_literals' + self.parser, "\t.section\t__TEXT,__literal16,16byte_literals" ).parameters, - ['__TEXT', '__literal16', '16byte_literals'], + ["__TEXT", "__literal16", "16byte_literals"], ) self.assertEqual( - self._get_directive(self.parser, '\t.align\t16,0x90').parameters[1], '0x90' + self._get_directive(self.parser, "\t.align\t16,0x90").parameters[1], "0x90" ) self.assertEqual( - self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[ - 'name' - ], - 'byte', + self._get_directive(self.parser, " .byte 100,103,144 #IACA START")["name"], + "byte", ) self.assertEqual( - self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[ - 'parameters' + self._get_directive(self.parser, " .byte 100,103,144 #IACA START")[ + "parameters" ][2], - '144', + "144", ) self.assertEqual( - ' '.join( - self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[ - 'comment' + " ".join( + self._get_directive(self.parser, " .byte 100,103,144 #IACA START")[ + "comment" ] ), - 'IACA START', + "IACA START", ) def test_parse_instruction(self): - instr1 = '\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27' - instr2 = 'jb ..B1.4 \t' - instr3 = ' movl $222,%ebx #IACA END' - instr4 = 'vmovss %xmm4, -4(%rsp,%rax,8) #12.9' - instr5 = 'mov %ebx,var(,1)' - instr6 = 'lea (,%rax,8),%rbx' - instr7 = 'vinsertf128 $0x1, %xmm0, %ymm1, %ymm1' + instr1 = "\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27" + instr2 = "jb ..B1.4 \t" + instr3 = " movl $222,%ebx #IACA END" + instr4 = "vmovss %xmm4, -4(%rsp,%rax,8) #12.9" + instr5 = "mov %ebx,var(,1)" + instr6 = "lea (,%rax,8),%rbx" + instr7 = "vinsertf128 $0x1, %xmm0, %ymm1, %ymm1" parsed_1 = self.parser.parse_instruction(instr1) parsed_2 = self.parser.parse_instruction(instr2) @@ -111,99 +109,99 @@ class TestParserX86ATT(unittest.TestCase): parsed_6 = self.parser.parse_instruction(instr6) parsed_7 = self.parser.parse_instruction(instr7) - self.assertEqual(parsed_1.instruction, 'vcvtsi2ss') - self.assertEqual(parsed_1.operands[0].register.name, 'edx') - self.assertEqual(parsed_1.operands[1].register.name, 'xmm2') - self.assertEqual(parsed_1.comment, '12.27') + self.assertEqual(parsed_1.instruction, "vcvtsi2ss") + self.assertEqual(parsed_1.operands[0].register.name, "edx") + self.assertEqual(parsed_1.operands[1].register.name, "xmm2") + self.assertEqual(parsed_1.comment, "12.27") - self.assertEqual(parsed_2.instruction, 'jb') - self.assertEqual(parsed_2.operands[0].identifier.name, '..B1.4') + self.assertEqual(parsed_2.instruction, "jb") + self.assertEqual(parsed_2.operands[0].identifier.name, "..B1.4") self.assertEqual(len(parsed_2.operands), 1) self.assertIsNone(parsed_2.comment) - self.assertEqual(parsed_3.instruction, 'movl') - self.assertEqual(parsed_3.operands[0].immediate.value, '222') - self.assertEqual(parsed_3.operands[1].register.name, 'ebx') - self.assertEqual(parsed_3.comment, 'IACA END') + self.assertEqual(parsed_3.instruction, "movl") + self.assertEqual(parsed_3.operands[0].immediate.value, "222") + self.assertEqual(parsed_3.operands[1].register.name, "ebx") + self.assertEqual(parsed_3.comment, "IACA END") - self.assertEqual(parsed_4.instruction, 'vmovss') - self.assertEqual(parsed_4.operands[1].memory.offset.value, '-4') - self.assertEqual(parsed_4.operands[1].memory.base.name, 'rsp') - self.assertEqual(parsed_4.operands[1].memory.index.name, 'rax') + self.assertEqual(parsed_4.instruction, "vmovss") + self.assertEqual(parsed_4.operands[1].memory.offset.value, "-4") + self.assertEqual(parsed_4.operands[1].memory.base.name, "rsp") + self.assertEqual(parsed_4.operands[1].memory.index.name, "rax") self.assertEqual(parsed_4.operands[1].memory.scale, 8) - self.assertEqual(parsed_4.operands[0].register.name, 'xmm4') - self.assertEqual(parsed_4.comment, '12.9') + self.assertEqual(parsed_4.operands[0].register.name, "xmm4") + self.assertEqual(parsed_4.comment, "12.9") - self.assertEqual(parsed_5.instruction, 'mov') - self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, 'var') + self.assertEqual(parsed_5.instruction, "mov") + self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, "var") self.assertIsNone(parsed_5.operands[1].memory.base) self.assertIsNone(parsed_5.operands[1].memory.index) self.assertEqual(parsed_5.operands[1].memory.scale, 1) - self.assertEqual(parsed_5.operands[0].register.name, 'ebx') + self.assertEqual(parsed_5.operands[0].register.name, "ebx") - self.assertEqual(parsed_6.instruction, 'lea') + self.assertEqual(parsed_6.instruction, "lea") self.assertIsNone(parsed_6.operands[0].memory.offset) self.assertIsNone(parsed_6.operands[0].memory.base) - self.assertEqual(parsed_6.operands[0].memory.index.name, 'rax') + self.assertEqual(parsed_6.operands[0].memory.index.name, "rax") self.assertEqual(parsed_6.operands[0].memory.scale, 8) - self.assertEqual(parsed_6.operands[1].register.name, 'rbx') + self.assertEqual(parsed_6.operands[1].register.name, "rbx") - self.assertEqual(parsed_7.operands[0].immediate.value, '0x1') - self.assertEqual(parsed_7.operands[1].register.name, 'xmm0') - self.assertEqual(parsed_7.operands[2].register.name, 'ymm1') - self.assertEqual(parsed_7.operands[3].register.name, 'ymm1') + self.assertEqual(parsed_7.operands[0].immediate.value, "0x1") + self.assertEqual(parsed_7.operands[1].register.name, "xmm0") + self.assertEqual(parsed_7.operands[2].register.name, "ymm1") + self.assertEqual(parsed_7.operands[3].register.name, "ymm1") def test_parse_line(self): - line_comment = '# -- Begin main' - line_label = '..B1.7: # Preds ..B1.6' - line_directive = '.quad .2.3_2__kmpc_loc_pack.2 #qed' - line_instruction = 'lea 2(%rax,%rax), %ecx #12.9' + line_comment = "# -- Begin main" + line_label = "..B1.7: # Preds ..B1.6" + line_directive = ".quad .2.3_2__kmpc_loc_pack.2 #qed" + line_instruction = "lea 2(%rax,%rax), %ecx #12.9" instruction_form_1 = { - 'instruction': None, - 'operands': [], - 'directive': None, - 'comment': '-- Begin main', - 'label': None, - 'line': '# -- Begin main', - 'line_number': 1, + "instruction": None, + "operands": [], + "directive": None, + "comment": "-- Begin main", + "label": None, + "line": "# -- Begin main", + "line_number": 1, } instruction_form_2 = { - 'instruction': None, - 'operands': [], - 'directive': None, - 'comment': 'Preds ..B1.6', - 'label': '..B1.7', - 'line': '..B1.7: # Preds ..B1.6', - 'line_number': 2, + "instruction": None, + "operands": [], + "directive": None, + "comment": "Preds ..B1.6", + "label": "..B1.7", + "line": "..B1.7: # Preds ..B1.6", + "line_number": 2, } instruction_form_3 = { - 'instruction': None, - 'operands': [], - 'directive': {'name': 'quad', 'parameters': ['.2.3_2__kmpc_loc_pack.2']}, - 'comment': 'qed', - 'label': None, - 'line': '.quad .2.3_2__kmpc_loc_pack.2 #qed', - 'line_number': 3, + "instruction": None, + "operands": [], + "directive": {"name": "quad", "parameters": [".2.3_2__kmpc_loc_pack.2"]}, + "comment": "qed", + "label": None, + "line": ".quad .2.3_2__kmpc_loc_pack.2 #qed", + "line_number": 3, } instruction_form_4 = { - 'instruction': 'lea', - 'operands': [ + "instruction": "lea", + "operands": [ { - 'memory': { - 'offset': {'value': '2'}, - 'base': {'name': 'rax'}, - 'index': {'name': 'rax'}, - 'scale': 1, + "memory": { + "offset": {"value": "2"}, + "base": {"name": "rax"}, + "index": {"name": "rax"}, + "scale": 1, } }, - {'register': {'name': 'ecx'}}, + {"register": {"name": "ecx"}}, ], - 'directive': None, - 'comment': '12.9', - 'label': None, - 'line': 'lea 2(%rax,%rax), %ecx #12.9', - 'line_number': 4, + "directive": None, + "comment": "12.9", + "label": None, + "line": "lea 2(%rax,%rax), %ecx #12.9", + "line_number": 4, } parsed_1 = self.parser.parse_line(line_comment, 1) @@ -222,27 +220,27 @@ class TestParserX86ATT(unittest.TestCase): self.assertEqual(len(parsed), 353) def test_parse_register(self): - register_str_1 = '%rax' - register_str_2 = '%r9' - register_str_3 = '%xmm1' - register_str_4 = '%rip' + register_str_1 = "%rax" + register_str_2 = "%r9" + register_str_3 = "%xmm1" + register_str_4 = "%rip" - parsed_reg_1 = {'register': {'name': 'rax'}} - parsed_reg_2 = {'register': {'name': 'r9'}} - parsed_reg_3 = {'register': {'name': 'xmm1'}} - parsed_reg_4 = {'register': {'name': 'rip'}} + parsed_reg_1 = {"register": {"name": "rax"}} + parsed_reg_2 = {"register": {"name": "r9"}} + parsed_reg_3 = {"register": {"name": "xmm1"}} + parsed_reg_4 = {"register": {"name": "rip"}} self.assertEqual(self.parser.parse_register(register_str_1), parsed_reg_1) self.assertEqual(self.parser.parse_register(register_str_2), parsed_reg_2) self.assertEqual(self.parser.parse_register(register_str_3), parsed_reg_3) self.assertEqual(self.parser.parse_register(register_str_4), parsed_reg_4) - self.assertIsNone(self.parser.parse_register('rax')) + self.assertIsNone(self.parser.parse_register("rax")) def test_normalize_imd(self): - imd_decimal_1 = {'value': '79'} - imd_hex_1 = {'value': '0x4f'} - imd_decimal_2 = {'value': '8'} - imd_hex_2 = {'value': '0x8'} + imd_decimal_1 = {"value": "79"} + imd_hex_1 = {"value": "0x4f"} + imd_decimal_2 = {"value": "8"} + imd_hex_2 = {"value": "0x8"} self.assertEqual( self.parser.normalize_imd(imd_decimal_1), self.parser.normalize_imd(imd_hex_1) ) @@ -251,22 +249,22 @@ class TestParserX86ATT(unittest.TestCase): ) def test_reg_dependency(self): - reg_a1 = AttrDict({'name': 'rax'}) - reg_a2 = AttrDict({'name': 'eax'}) - reg_a3 = AttrDict({'name': 'ax'}) - reg_a4 = AttrDict({'name': 'al'}) - reg_r11 = AttrDict({'name': 'r11'}) - reg_r11b = AttrDict({'name': 'r11b'}) - reg_r11d = AttrDict({'name': 'r11d'}) - reg_r11w = AttrDict({'name': 'r11w'}) - reg_xmm1 = AttrDict({'name': 'xmm1'}) - reg_ymm1 = AttrDict({'name': 'ymm1'}) - reg_zmm1 = AttrDict({'name': 'zmm1'}) + reg_a1 = AttrDict({"name": "rax"}) + reg_a2 = AttrDict({"name": "eax"}) + reg_a3 = AttrDict({"name": "ax"}) + reg_a4 = AttrDict({"name": "al"}) + reg_r11 = AttrDict({"name": "r11"}) + reg_r11b = AttrDict({"name": "r11b"}) + reg_r11d = AttrDict({"name": "r11d"}) + reg_r11w = AttrDict({"name": "r11w"}) + reg_xmm1 = AttrDict({"name": "xmm1"}) + reg_ymm1 = AttrDict({"name": "ymm1"}) + reg_zmm1 = AttrDict({"name": "zmm1"}) - reg_b1 = AttrDict({'name': 'rbx'}) - reg_r15 = AttrDict({'name': 'r15'}) - reg_xmm2 = AttrDict({'name': 'xmm2'}) - reg_ymm3 = AttrDict({'name': 'ymm3'}) + reg_b1 = AttrDict({"name": "rbx"}) + reg_r15 = AttrDict({"name": "r15"}) + reg_xmm2 = AttrDict({"name": "xmm2"}) + reg_ymm3 = AttrDict({"name": "ymm3"}) reg_a = [reg_a1, reg_a2, reg_a3, reg_a4] reg_r = [reg_r11, reg_r11b, reg_r11d, reg_r11w] @@ -300,7 +298,7 @@ class TestParserX86ATT(unittest.TestCase): # Helper functions ################## def _get_comment(self, parser, comment): - return ' '.join( + return " ".join( AttrDict.convert_dict( parser.process_operand(parser.comment.parseString(comment, parseAll=True).asDict()) ).comment @@ -319,11 +317,11 @@ class TestParserX86ATT(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestParserX86ATT) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_semantics.py b/tests/test_semantics.py index 08cce4e..1e24820 100755 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -12,13 +12,12 @@ import networkx as nx from osaca.osaca import get_unmatched_instruction_ratio from osaca.parser import AttrDict, ParserAArch64, ParserX86ATT -from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG, - MachineModel, reduce_to_section) +from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section class TestSemanticTools(unittest.TestCase): MODULE_DATA_DIR = os.path.join( - os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' + os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), "osaca/data/" ) @classmethod @@ -26,30 +25,30 @@ class TestSemanticTools(unittest.TestCase): # set up parser and kernels cls.parser_x86 = ParserX86ATT() cls.parser_AArch64 = ParserAArch64() - with open(cls._find_file('kernel_x86.s')) as f: + with open(cls._find_file("kernel_x86.s")) as f: cls.code_x86 = f.read() - with open(cls._find_file('kernel_aarch64.s')) as f: + with open(cls._find_file("kernel_aarch64.s")) as f: cls.code_AArch64 = f.read() - cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), 'x86') + cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), "x86") cls.kernel_AArch64 = reduce_to_section( - cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64' + cls.parser_AArch64.parse_file(cls.code_AArch64), "aarch64" ) # set up machine models cls.machine_model_csx = MachineModel( - path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml') + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "csx.yml") ) cls.machine_model_tx2 = MachineModel( - path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml') + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "tx2.yml") ) cls.semantics_csx = ArchSemantics( - cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/x86.yml') + cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/x86.yml") ) cls.semantics_tx2 = ArchSemantics( cls.machine_model_tx2, - path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'), + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/aarch64.yml"), ) - cls.machine_model_zen = MachineModel(arch='zen1') + cls.machine_model_zen = MachineModel(arch="zen1") for i in range(len(cls.kernel_x86)): cls.semantics_csx.assign_src_dst(cls.kernel_x86[i]) @@ -64,7 +63,7 @@ class TestSemanticTools(unittest.TestCase): def test_creation_by_name(self): try: - tmp_mm = MachineModel(arch='CSX') + tmp_mm = MachineModel(arch="CSX") ArchSemantics(tmp_mm) except ValueError: self.fail() @@ -72,12 +71,12 @@ class TestSemanticTools(unittest.TestCase): def test_machine_model_various_functions(self): # check dummy MachineModel creation try: - MachineModel(isa='x86') - MachineModel(isa='aarch64') + MachineModel(isa="x86") + MachineModel(isa="aarch64") except ValueError: self.fail() - test_mm_x86 = MachineModel(path_to_yaml=self._find_file('test_db_x86.yml')) - test_mm_arm = MachineModel(path_to_yaml=self._find_file('test_db_aarch64.yml')) + test_mm_x86 = MachineModel(path_to_yaml=self._find_file("test_db_x86.yml")) + test_mm_arm = MachineModel(path_to_yaml=self._find_file("test_db_aarch64.yml")) # test get_instruction without mnemonic self.assertIsNone(test_mm_x86.get_instruction(None, [])) @@ -86,80 +85,80 @@ class TestSemanticTools(unittest.TestCase): # test get_instruction from DB self.assertIsNone(test_mm_x86.get_instruction(None, [])) self.assertIsNone(test_mm_arm.get_instruction(None, [])) - self.assertIsNone(test_mm_x86.get_instruction('NOT_IN_DB', [])) - self.assertIsNone(test_mm_arm.get_instruction('NOT_IN_DB', [])) - name_x86_1 = 'vaddpd' + self.assertIsNone(test_mm_x86.get_instruction("NOT_IN_DB", [])) + self.assertIsNone(test_mm_arm.get_instruction("NOT_IN_DB", [])) + name_x86_1 = "vaddpd" operands_x86_1 = [ - {'class': 'register', 'name': 'xmm'}, - {'class': 'register', 'name': 'xmm'}, - {'class': 'register', 'name': 'xmm'}, + {"class": "register", "name": "xmm"}, + {"class": "register", "name": "xmm"}, + {"class": "register", "name": "xmm"}, ] instr_form_x86_1 = test_mm_x86.get_instruction(name_x86_1, operands_x86_1) self.assertEqual(instr_form_x86_1, test_mm_x86.get_instruction(name_x86_1, operands_x86_1)) self.assertEqual( - test_mm_x86.get_instruction('jg', [{'class': 'identifier'}]), - test_mm_x86.get_instruction('jg', [{'class': 'identifier'}]), + test_mm_x86.get_instruction("jg", [{"class": "identifier"}]), + test_mm_x86.get_instruction("jg", [{"class": "identifier"}]), ) - name_arm_1 = 'fadd' + name_arm_1 = "fadd" operands_arm_1 = [ - {'class': 'register', 'prefix': 'v', 'shape': 's'}, - {'class': 'register', 'prefix': 'v', 'shape': 's'}, - {'class': 'register', 'prefix': 'v', 'shape': 's'}, + {"class": "register", "prefix": "v", "shape": "s"}, + {"class": "register", "prefix": "v", "shape": "s"}, + {"class": "register", "prefix": "v", "shape": "s"}, ] instr_form_arm_1 = test_mm_arm.get_instruction(name_arm_1, operands_arm_1) self.assertEqual(instr_form_arm_1, test_mm_arm.get_instruction(name_arm_1, operands_arm_1)) self.assertEqual( - test_mm_arm.get_instruction('b.ne', [{'class': 'identifier'}]), - test_mm_arm.get_instruction('b.ne', [{'class': 'identifier'}]), + test_mm_arm.get_instruction("b.ne", [{"class": "identifier"}]), + test_mm_arm.get_instruction("b.ne", [{"class": "identifier"}]), ) # test full instruction name self.assertEqual( MachineModel.get_full_instruction_name(instr_form_x86_1), - 'vaddpd register(name:xmm),register(name:xmm),register(name:xmm)', + "vaddpd register(name:xmm),register(name:xmm),register(name:xmm)", ) self.assertEqual( MachineModel.get_full_instruction_name(instr_form_arm_1), - 'fadd register(prefix:v,shape:s),register(prefix:v,shape:s),' - + 'register(prefix:v,shape:s)', + "fadd register(prefix:v,shape:s),register(prefix:v,shape:s)," + + "register(prefix:v,shape:s)", ) # test get_store_tp self.assertEqual( test_mm_x86.get_store_throughput( - {'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': 1} + {"base": {"name": "x"}, "offset": None, "index": None, "scale": 1} ), - [[2, '237'], [2, '4']], + [[2, "237"], [2, "4"]], ) self.assertEqual( test_mm_x86.get_store_throughput( - {'base': {'prefix': 'NOT_IN_DB'}, 'offset': None, 'index': 'NOT_NONE', 'scale': 1} + {"base": {"prefix": "NOT_IN_DB"}, "offset": None, "index": "NOT_NONE", "scale": 1} ), - [[1, '23'], [1, '4']], + [[1, "23"], [1, "4"]], ) self.assertEqual( test_mm_arm.get_store_throughput( - {'base': {'prefix': 'x'}, 'offset': None, 'index': None, 'scale': 1} + {"base": {"prefix": "x"}, "offset": None, "index": None, "scale": 1} ), - [[2, '34'], [2, '5']], + [[2, "34"], [2, "5"]], ) self.assertEqual( test_mm_arm.get_store_throughput( - {'base': {'prefix': 'NOT_IN_DB'}, 'offset': None, 'index': None, 'scale': 1} + {"base": {"prefix": "NOT_IN_DB"}, "offset": None, "index": None, "scale": 1} ), - [[1, '34'], [1, '5']], + [[1, "34"], [1, "5"]], ) # test get_store_lt self.assertEqual( test_mm_x86.get_store_latency( - {'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': '1'} + {"base": {"name": "x"}, "offset": None, "index": None, "scale": "1"} ), 0, ) self.assertEqual( test_mm_arm.get_store_latency( - {'base': {'prefix': 'x'}, 'offset': None, 'index': None, 'scale': '1'} + {"base": {"prefix": "x"}, "offset": None, "index": None, "scale": "1"} ), 0, ) @@ -170,55 +169,55 @@ class TestSemanticTools(unittest.TestCase): # test default load tp self.assertEqual( test_mm_x86.get_load_throughput( - {'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': 1} + {"base": {"name": "x"}, "offset": None, "index": None, "scale": 1} ), - [[1, '23'], [1, ['2D', '3D']]], + [[1, "23"], [1, ["2D", "3D"]]], ) # test adding port - test_mm_x86.add_port('dummyPort') - test_mm_arm.add_port('dummyPort') + test_mm_x86.add_port("dummyPort") + test_mm_arm.add_port("dummyPort") # test dump of DB - with open('/dev/null', 'w') as dev_null: + with open("/dev/null", "w") as dev_null: test_mm_x86.dump(stream=dev_null) test_mm_arm.dump(stream=dev_null) def test_src_dst_assignment_x86(self): for instruction_form in self.kernel_x86: with self.subTest(instruction_form=instruction_form): - if instruction_form['semantic_operands'] is not None: - self.assertTrue('source' in instruction_form['semantic_operands']) - self.assertTrue('destination' in instruction_form['semantic_operands']) - self.assertTrue('src_dst' in instruction_form['semantic_operands']) + if instruction_form["semantic_operands"] is not None: + self.assertTrue("source" in instruction_form["semantic_operands"]) + self.assertTrue("destination" in instruction_form["semantic_operands"]) + self.assertTrue("src_dst" in instruction_form["semantic_operands"]) def test_src_dst_assignment_AArch64(self): for instruction_form in self.kernel_AArch64: with self.subTest(instruction_form=instruction_form): - if instruction_form['semantic_operands'] is not None: - self.assertTrue('source' in instruction_form['semantic_operands']) - self.assertTrue('destination' in instruction_form['semantic_operands']) - self.assertTrue('src_dst' in instruction_form['semantic_operands']) + if instruction_form["semantic_operands"] is not None: + self.assertTrue("source" in instruction_form["semantic_operands"]) + self.assertTrue("destination" in instruction_form["semantic_operands"]) + self.assertTrue("src_dst" in instruction_form["semantic_operands"]) def test_tp_lt_assignment_x86(self): - self.assertTrue('ports' in self.machine_model_csx) - port_num = len(self.machine_model_csx['ports']) + self.assertTrue("ports" in self.machine_model_csx) + port_num = len(self.machine_model_csx["ports"]) for instruction_form in self.kernel_x86: with self.subTest(instruction_form=instruction_form): - self.assertTrue('throughput' in instruction_form) - self.assertTrue('latency' in instruction_form) - self.assertIsInstance(instruction_form['port_pressure'], list) - self.assertEqual(len(instruction_form['port_pressure']), port_num) + self.assertTrue("throughput" in instruction_form) + self.assertTrue("latency" in instruction_form) + self.assertIsInstance(instruction_form["port_pressure"], list) + self.assertEqual(len(instruction_form["port_pressure"]), port_num) def test_tp_lt_assignment_AArch64(self): - self.assertTrue('ports' in self.machine_model_tx2) - port_num = len(self.machine_model_tx2['ports']) + self.assertTrue("ports" in self.machine_model_tx2) + port_num = len(self.machine_model_tx2["ports"]) for instruction_form in self.kernel_AArch64: with self.subTest(instruction_form=instruction_form): - self.assertTrue('throughput' in instruction_form) - self.assertTrue('latency' in instruction_form) - self.assertIsInstance(instruction_form['port_pressure'], list) - self.assertEqual(len(instruction_form['port_pressure']), port_num) + self.assertTrue("throughput" in instruction_form) + self.assertTrue("latency" in instruction_form) + self.assertIsInstance(instruction_form["port_pressure"], list) + self.assertEqual(len(instruction_form["port_pressure"]), port_num) def test_optimal_throughput_assignment(self): # x86 @@ -266,7 +265,7 @@ class TestSemanticTools(unittest.TestCase): with self.assertRaises(ValueError): dg.get_dependent_instruction_forms() # test dot creation - dg.export_graph(filepath='/dev/null') + dg.export_graph(filepath="/dev/null") def test_kernelDG_AArch64(self): dg = KernelDG(self.kernel_AArch64, self.parser_AArch64, self.machine_model_tx2) @@ -292,11 +291,11 @@ class TestSemanticTools(unittest.TestCase): with self.assertRaises(ValueError): dg.get_dependent_instruction_forms() # test dot creation - dg.export_graph(filepath='/dev/null') + dg.export_graph(filepath="/dev/null") def test_hidden_load(self): machine_model_hld = MachineModel( - path_to_yaml=self._find_file('hidden_load_machine_model.yml') + path_to_yaml=self._find_file("hidden_load_machine_model.yml") ) self.assertTrue(machine_model_hld.has_hidden_loads()) semantics_hld = ArchSemantics(machine_model_hld) @@ -308,9 +307,9 @@ class TestSemanticTools(unittest.TestCase): semantics_hld.add_semantics(kernel_hld_2) semantics_hld.add_semantics(kernel_hld_3) - num_hidden_loads = len([x for x in kernel_hld if INSTR_FLAGS.HIDDEN_LD in x['flags']]) - num_hidden_loads_2 = len([x for x in kernel_hld_2 if INSTR_FLAGS.HIDDEN_LD in x['flags']]) - num_hidden_loads_3 = len([x for x in kernel_hld_3 if INSTR_FLAGS.HIDDEN_LD in x['flags']]) + num_hidden_loads = len([x for x in kernel_hld if INSTR_FLAGS.HIDDEN_LD in x["flags"]]) + num_hidden_loads_2 = len([x for x in kernel_hld_2 if INSTR_FLAGS.HIDDEN_LD in x["flags"]]) + num_hidden_loads_3 = len([x for x in kernel_hld_3 if INSTR_FLAGS.HIDDEN_LD in x["flags"]]) self.assertEqual(num_hidden_loads, 1) self.assertEqual(num_hidden_loads_2, 0) self.assertEqual(num_hidden_loads_3, 1) @@ -333,42 +332,42 @@ class TestSemanticTools(unittest.TestCase): self.assertEqual(len(lc_deps), 2) # ID 8 self.assertEqual( - lc_deps[lcd_id]['root'], dg.dg.nodes(data=True)[lcd_id]['instruction_form'] + lc_deps[lcd_id]["root"], dg.dg.nodes(data=True)[lcd_id]["instruction_form"] ) - self.assertEqual(len(lc_deps[lcd_id]['dependencies']), 1) + self.assertEqual(len(lc_deps[lcd_id]["dependencies"]), 1) self.assertEqual( - lc_deps[lcd_id]['dependencies'][0], dg.dg.nodes(data=True)[lcd_id]['instruction_form'] + lc_deps[lcd_id]["dependencies"][0], dg.dg.nodes(data=True)[lcd_id]["instruction_form"] ) # w/ flag dependencies: ID 9 w/ len=2 # w/o flag dependencies: ID 5 w/ len=1 # TODO discuss self.assertEqual( - lc_deps[lcd_id2]['root'], dg.dg.nodes(data=True)[lcd_id2]['instruction_form'] + lc_deps[lcd_id2]["root"], dg.dg.nodes(data=True)[lcd_id2]["instruction_form"] ) - self.assertEqual(len(lc_deps[lcd_id2]['dependencies']), 1) + self.assertEqual(len(lc_deps[lcd_id2]["dependencies"]), 1) self.assertEqual( - lc_deps[lcd_id2]['dependencies'][0], - dg.dg.nodes(data=True)[lcd_id2]['instruction_form'], + lc_deps[lcd_id2]["dependencies"][0], + dg.dg.nodes(data=True)[lcd_id2]["instruction_form"], ) def test_is_read_is_written_x86(self): # independent form HW model dag = KernelDG(self.kernel_x86, self.parser_x86, None) - reg_rcx = AttrDict({'name': 'rcx'}) - reg_ymm1 = AttrDict({'name': 'ymm1'}) + reg_rcx = AttrDict({"name": "rcx"}) + reg_ymm1 = AttrDict({"name": "ymm1"}) - instr_form_r_c = self.parser_x86.parse_line('vmovsd %xmm0, (%r15,%rcx,8)') + instr_form_r_c = self.parser_x86.parse_line("vmovsd %xmm0, (%r15,%rcx,8)") self.semantics_csx.assign_src_dst(instr_form_r_c) - instr_form_non_r_c = self.parser_x86.parse_line('movl %xmm0, (%r15,%rax,8)') + instr_form_non_r_c = self.parser_x86.parse_line("movl %xmm0, (%r15,%rax,8)") self.semantics_csx.assign_src_dst(instr_form_non_r_c) - instr_form_w_c = self.parser_x86.parse_line('movi $0x05ACA, %rcx') + instr_form_w_c = self.parser_x86.parse_line("movi $0x05ACA, %rcx") self.semantics_csx.assign_src_dst(instr_form_w_c) - instr_form_rw_ymm_1 = self.parser_x86.parse_line('vinsertf128 $0x1, %xmm1, %ymm0, %ymm1') + instr_form_rw_ymm_1 = self.parser_x86.parse_line("vinsertf128 $0x1, %xmm1, %ymm0, %ymm1") self.semantics_csx.assign_src_dst(instr_form_rw_ymm_1) - instr_form_rw_ymm_2 = self.parser_x86.parse_line('vinsertf128 $0x1, %xmm0, %ymm1, %ymm1') + instr_form_rw_ymm_2 = self.parser_x86.parse_line("vinsertf128 $0x1, %xmm0, %ymm1, %ymm1") self.semantics_csx.assign_src_dst(instr_form_rw_ymm_2) - instr_form_r_ymm = self.parser_x86.parse_line('vmovapd %ymm1, %ymm0') + instr_form_r_ymm = self.parser_x86.parse_line("vmovapd %ymm1, %ymm0") self.semantics_csx.assign_src_dst(instr_form_r_ymm) self.assertTrue(dag.is_read(reg_rcx, instr_form_r_c)) @@ -387,29 +386,29 @@ class TestSemanticTools(unittest.TestCase): def test_is_read_is_written_AArch64(self): # independent form HW model dag = KernelDG(self.kernel_AArch64, self.parser_AArch64, None) - reg_x1 = AttrDict({'prefix': 'x', 'name': '1'}) - reg_w1 = AttrDict({'prefix': 'w', 'name': '1'}) - reg_d1 = AttrDict({'prefix': 'd', 'name': '1'}) - reg_q1 = AttrDict({'prefix': 'q', 'name': '1'}) - reg_v1 = AttrDict({'prefix': 'v', 'name': '1', 'lanes': '2', 'shape': 'd'}) + reg_x1 = AttrDict({"prefix": "x", "name": "1"}) + reg_w1 = AttrDict({"prefix": "w", "name": "1"}) + reg_d1 = AttrDict({"prefix": "d", "name": "1"}) + reg_q1 = AttrDict({"prefix": "q", "name": "1"}) + reg_v1 = AttrDict({"prefix": "v", "name": "1", "lanes": "2", "shape": "d"}) regs = [reg_d1, reg_q1, reg_v1] regs_gp = [reg_w1, reg_x1] - instr_form_r_1 = self.parser_AArch64.parse_line('stp q1, q3, [x12, #192]') + instr_form_r_1 = self.parser_AArch64.parse_line("stp q1, q3, [x12, #192]") self.semantics_tx2.assign_src_dst(instr_form_r_1) - instr_form_r_2 = self.parser_AArch64.parse_line('fadd v2.2d, v1.2d, v0.2d') + instr_form_r_2 = self.parser_AArch64.parse_line("fadd v2.2d, v1.2d, v0.2d") self.semantics_tx2.assign_src_dst(instr_form_r_2) - instr_form_w_1 = self.parser_AArch64.parse_line('ldr d1, [x1, #:got_lo12:q2c]') + instr_form_w_1 = self.parser_AArch64.parse_line("ldr d1, [x1, #:got_lo12:q2c]") self.semantics_tx2.assign_src_dst(instr_form_w_1) - instr_form_non_w_1 = self.parser_AArch64.parse_line('ldr x1, [x1, #:got_lo12:q2c]') + instr_form_non_w_1 = self.parser_AArch64.parse_line("ldr x1, [x1, #:got_lo12:q2c]") self.semantics_tx2.assign_src_dst(instr_form_non_w_1) - instr_form_rw_1 = self.parser_AArch64.parse_line('fmul v1.2d, v1.2d, v0.2d') + instr_form_rw_1 = self.parser_AArch64.parse_line("fmul v1.2d, v1.2d, v0.2d") self.semantics_tx2.assign_src_dst(instr_form_rw_1) - instr_form_rw_2 = self.parser_AArch64.parse_line('ldp q2, q4, [x1, #64]!') + instr_form_rw_2 = self.parser_AArch64.parse_line("ldp q2, q4, [x1, #64]!") self.semantics_tx2.assign_src_dst(instr_form_rw_2) - instr_form_rw_3 = self.parser_AArch64.parse_line('str x4, [x1], #64') + instr_form_rw_3 = self.parser_AArch64.parse_line("str x4, [x1], #64") self.semantics_tx2.assign_src_dst(instr_form_rw_3) - instr_form_non_rw_1 = self.parser_AArch64.parse_line('adds x1, x11') + instr_form_non_rw_1 = self.parser_AArch64.parse_line("adds x1, x11") self.semantics_tx2.assign_src_dst(instr_form_non_rw_1) for reg in regs: @@ -447,43 +446,43 @@ class TestSemanticTools(unittest.TestCase): with self.assertRaises(ValueError): MachineModel() with self.assertRaises(ValueError): - MachineModel(arch='CSX', path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')) + MachineModel(arch="CSX", path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml")) with self.assertRaises(FileNotFoundError): - MachineModel(arch='THE_MACHINE') + MachineModel(arch="THE_MACHINE") with self.assertRaises(FileNotFoundError): - MachineModel(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'THE_MACHINE.yml')) + MachineModel(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "THE_MACHINE.yml")) def test_MachineModel_getter(self): sample_operands = [ { - 'memory': { - 'offset': None, - 'base': {'name': 'r12'}, - 'index': {'name': 'rcx'}, - 'scale': 8, + "memory": { + "offset": None, + "base": {"name": "r12"}, + "index": {"name": "rcx"}, + "scale": 8, } } ] - self.assertIsNone(self.machine_model_csx.get_instruction('GETRESULT', sample_operands)) - self.assertIsNone(self.machine_model_tx2.get_instruction('GETRESULT', sample_operands)) + self.assertIsNone(self.machine_model_csx.get_instruction("GETRESULT", sample_operands)) + self.assertIsNone(self.machine_model_tx2.get_instruction("GETRESULT", sample_operands)) - self.assertEqual(self.machine_model_csx.get_arch(), 'csx') - self.assertEqual(self.machine_model_tx2.get_arch(), 'tx2') + self.assertEqual(self.machine_model_csx.get_arch(), "csx") + self.assertEqual(self.machine_model_tx2.get_arch(), "tx2") - self.assertEqual(self.machine_model_csx.get_ISA(), 'x86') - self.assertEqual(self.machine_model_tx2.get_ISA(), 'aarch64') + self.assertEqual(self.machine_model_csx.get_ISA(), "x86") + self.assertEqual(self.machine_model_tx2.get_ISA(), "aarch64") - ports_csx = ['0', '0DV', '1', '2', '2D', '3', '3D', '4', '5', '6', '7'] - data_ports_csx = ['2D', '3D'] + ports_csx = ["0", "0DV", "1", "2", "2D", "3", "3D", "4", "5", "6", "7"] + data_ports_csx = ["2D", "3D"] self.assertEqual(self.machine_model_csx.get_ports(), ports_csx) self.assertEqual(self.machine_model_csx.get_data_ports(), data_ports_csx) self.assertFalse(self.machine_model_tx2.has_hidden_loads()) - self.assertEqual(MachineModel.get_isa_for_arch('CSX'), 'x86') - self.assertEqual(MachineModel.get_isa_for_arch('tX2'), 'aarch64') + self.assertEqual(MachineModel.get_isa_for_arch("CSX"), "x86") + self.assertEqual(MachineModel.get_isa_for_arch("tX2"), "aarch64") with self.assertRaises(ValueError): - self.assertIsNone(MachineModel.get_isa_for_arch('THE_MACHINE')) + self.assertIsNone(MachineModel.get_isa_for_arch("THE_MACHINE")) ################## # Helper functions @@ -492,11 +491,11 @@ class TestSemanticTools(unittest.TestCase): @staticmethod def _find_file(name): testdir = os.path.dirname(__file__) - name = os.path.join(testdir, 'test_files', name) + name = os.path.join(testdir, "test_files", name) assert os.path.exists(name) return name -if __name__ == '__main__': +if __name__ == "__main__": suite = unittest.TestLoader().loadTestsFromTestCase(TestSemanticTools) unittest.TextTestRunner(verbosity=2).run(suite)