migrate code style to Black

This commit is contained in:
Julian Hammer
2021-03-11 12:02:45 +01:00
parent 1ebe5ecfbd
commit 6204c90934
33 changed files with 3045 additions and 2960 deletions

View File

@@ -8,15 +8,15 @@ import os
import sys
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath("."))
from version_from_src import get_version
# -- Project information -----------------------------------------------------
project = 'OSACA'
copyright = '2020, Jan Laukemann'
author = 'Jan Laukemann'
html_logo = 'img/osaca-logo.png'
project = "OSACA"
copyright = "2020, Jan Laukemann"
author = "Jan Laukemann"
html_logo = "img/osaca-logo.png"
# The full version, including alpha/beta/rc tags
version = get_version()
@@ -28,21 +28,21 @@ release = get_version()
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.mathjax',
'sphinx.ext.napoleon',
'sphinx.ext.todo',
'sphinx.ext.viewcode',
"sphinx.ext.autodoc",
"sphinx.ext.doctest",
"sphinx.ext.intersphinx",
"sphinx.ext.mathjax",
"sphinx.ext.napoleon",
"sphinx.ext.todo",
"sphinx.ext.viewcode",
]
add_module_names = False
source_suffix = '.rst'
master_doc = 'index'
source_suffix = ".rst"
master_doc = "index"
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
@@ -55,13 +55,13 @@ exclude_patterns = []
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
# e.g., 'alabaster', 'sphinx_rtd_theme'
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
htmlhelp_basename = 'osaca_doc'
html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
htmlhelp_basename = "osaca_doc"
html_sidebars = {"**": ["globaltoc.html", "relations.html", "sourcelink.html", "searchbox.html"]}
autodoc_member_order = 'bysource'
autodoc_member_order = "bysource"

View File

@@ -19,7 +19,7 @@ def __find_version(*file_paths):
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
if version_match:
return version_match.group(1)
raise RuntimeError('Unable to find version string.')
raise RuntimeError("Unable to find version string.")
def get_version():
@@ -28,4 +28,4 @@ def get_version():
:returns: str -- the version string.
"""
return __find_version('../osaca/__init__.py')
return __find_version("../osaca/__init__.py")

View File

@@ -1,6 +1,6 @@
"""Open Source Architecture Code Analyzer"""
name = 'osaca'
__version__ = '0.3.14'
name = "osaca"
__version__ = "0.3.14"
# To trigger travis deployment to pypi, do the following:
# 1. Increment __version___

View File

@@ -5,4 +5,4 @@ Only the classes below will be exported, so please add new semantic tools to __a
"""
from .kerncraft_interface import KerncraftAPI
__all__ = ['KerncraftAPI']
__all__ = ["KerncraftAPI"]

View File

@@ -6,8 +6,7 @@ from io import StringIO
from osaca.frontend import Frontend
from osaca.parser import ParserAArch64, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, KernelDG, MachineModel,
ArchSemantics, reduce_to_section)
from osaca.semantics import INSTR_FLAGS, KernelDG, MachineModel, ArchSemantics, reduce_to_section
# Stolen from https://stackoverflow.com/a/16571630
@@ -28,9 +27,9 @@ class KerncraftAPI(object):
self.machine_model = MachineModel(arch=arch)
self.semantics = ArchSemantics(self.machine_model)
isa = self.machine_model.get_ISA().lower()
if isa == 'aarch64':
if isa == "aarch64":
self.parser = ParserAArch64()
elif isa == 'x86':
elif isa == "x86":
self.parser = ParserX86ATT()
parsed_code = self.parser.parse_file(code)
@@ -46,15 +45,15 @@ class KerncraftAPI(object):
unmatched_counter = 0
for instruction in self.kernel:
if (
INSTR_FLAGS.TP_UNKWN in instruction['flags']
and INSTR_FLAGS.LT_UNKWN in instruction['flags']
INSTR_FLAGS.TP_UNKWN in instruction["flags"]
and INSTR_FLAGS.LT_UNKWN in instruction["flags"]
):
unmatched_counter += 1
return unmatched_counter / len(self.kernel)
def get_port_occupation_cycles(self):
throughput_values = self.semantics.get_throughput_sum(self.kernel)
port_names = self.machine_model['ports']
port_names = self.machine_model["ports"]
return collections.OrderedDict(list(zip(port_names, throughput_values)))
def get_total_throughput(self):
@@ -66,13 +65,13 @@ class KerncraftAPI(object):
def get_cp(self):
kernel_graph = KernelDG(self.kernel, self.parser, self.machine_model)
kernel_cp = kernel_graph.get_critical_path()
return sum([x['latency_cp'] for x in kernel_cp])
return sum([x["latency_cp"] for x in kernel_cp])
def get_lcd(self):
kernel_graph = KernelDG(self.kernel, self.parser, self.machine_model)
lcd_dict = kernel_graph.get_loopcarried_dependencies()
lcd = 0.0
for dep in lcd_dict:
lcd_tmp = sum([x['latency_lcd'] for x in lcd_dict[dep]['dependencies']])
lcd_tmp = sum([x["latency_lcd"] for x in lcd_dict[dep]["dependencies"]])
lcd = lcd_tmp if lcd_tmp > lcd else lcd
return lcd

View File

@@ -2,30 +2,33 @@
from glob import glob
import os.path
import sys
sys.path[0:0] = ['../..']
sys.path[0:0] = ["../.."]
failed = False
try:
from osaca.semantics.hw_model import MachineModel
except ModuleNotFoundError:
print("Unable to import MachineModel, probably some dependency is not yet installed. SKIPPING. "
print(
"Unable to import MachineModel, probably some dependency is not yet installed. SKIPPING. "
"First run of OSACA may take a while to build caches, subsequent runs will be as fast as "
"ever.")
"ever."
)
sys.exit()
print('Building cache: ', end='')
print("Building cache: ", end="")
sys.stdout.flush()
# Iterating architectures
for f in glob(os.path.join(os.path.dirname(__file__), '*.yml')):
for f in glob(os.path.join(os.path.dirname(__file__), "*.yml")):
MachineModel(path_to_yaml=f)
print('.', end='')
print(".", end="")
sys.stdout.flush()
# Iterating ISAs
for f in glob(os.path.join(os.path.dirname(__file__), 'isa/*.yml')):
for f in glob(os.path.join(os.path.dirname(__file__), "isa/*.yml")):
MachineModel(path_to_yaml=f)
print('+', end='')
print("+", end="")
sys.stdout.flush()
print()

File diff suppressed because it is too large Load Diff

View File

@@ -9,37 +9,37 @@ from osaca.parser import get_parser
from osaca.semantics import MachineModel
intel_archs = [
'CON',
'WOL',
'NHM',
'WSM',
'SNB',
'IVB',
'HSW',
'BDW',
'SKL',
'SKX',
'KBL',
'CFL',
'CNL',
'ICL',
"CON",
"WOL",
"NHM",
"WSM",
"SNB",
"IVB",
"HSW",
"BDW",
"SKL",
"SKX",
"KBL",
"CFL",
"CNL",
"ICL",
]
amd_archs = ['ZEN1', 'ZEN+', 'ZEN2']
amd_archs = ["ZEN1", "ZEN+", "ZEN2"]
def port_pressure_from_tag_attributes(attrib):
# '1*p015+1*p1+1*p23+1*p4+3*p5' ->
# [[1, '015'], [1, '1'], [1, '23'], [1, '4'], [3, '5']]
port_occupation = []
for p in attrib['ports'].split('+'):
cycles, ports = p.split('*')
ports = ports.lstrip('p')
ports = ports.lstrip('FP')
for p in attrib["ports"].split("+"):
cycles, ports = p.split("*")
ports = ports.lstrip("p")
ports = ports.lstrip("FP")
port_occupation.append([int(cycles), ports])
# Also consider div on DIV pipeline
if 'div_cycles' in attrib:
port_occupation.append([int(attrib['div_cycles']), ['DIV']])
if "div_cycles" in attrib:
port_occupation.append([int(attrib["div_cycles"]), ["DIV"]])
return port_occupation
@@ -47,57 +47,57 @@ def port_pressure_from_tag_attributes(attrib):
def extract_paramters(instruction_tag, parser, isa):
# Extract parameter components
parameters = [] # used to store string representations
parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib['idx']))
parameter_tags = sorted(instruction_tag.findall("operand"), key=lambda p: int(p.attrib["idx"]))
for parameter_tag in parameter_tags:
parameter = {}
# Ignore parameters with suppressed=1
if int(parameter_tag.attrib.get('suppressed', '0')):
if int(parameter_tag.attrib.get("suppressed", "0")):
continue
p_type = parameter_tag.attrib['type']
if p_type == 'imm':
parameter['class'] = 'immediate'
parameter['imd'] = 'int'
p_type = parameter_tag.attrib["type"]
if p_type == "imm":
parameter["class"] = "immediate"
parameter["imd"] = "int"
parameters.append(parameter)
elif p_type == 'mem':
parameter['class'] = 'memory'
parameter['base'] = "*"
parameter['offset'] = "*"
parameter['index'] = "*"
parameter['scale'] = "*"
elif p_type == "mem":
parameter["class"] = "memory"
parameter["base"] = "*"
parameter["offset"] = "*"
parameter["index"] = "*"
parameter["scale"] = "*"
parameters.append(parameter)
elif p_type == 'reg':
parameter['class'] = 'register'
possible_regs = [parser.parse_register('%' + r) for r in parameter_tag.text.split(',')]
elif p_type == "reg":
parameter["class"] = "register"
possible_regs = [parser.parse_register("%" + r) for r in parameter_tag.text.split(",")]
if possible_regs[0] is None:
raise ValueError(
'Unknown register type for {} with {}.'.format(
"Unknown register type for {} with {}.".format(
parameter_tag.attrib, parameter_tag.text
)
)
if isa == 'x86':
if parser.is_vector_register(possible_regs[0]['register']):
possible_regs[0]['register']['name'] = possible_regs[0]['register'][
'name'
if isa == "x86":
if parser.is_vector_register(possible_regs[0]["register"]):
possible_regs[0]["register"]["name"] = possible_regs[0]["register"][
"name"
].lower()[:3]
if 'mask' in possible_regs[0]['register']:
possible_regs[0]['register']['mask'] = True
if "mask" in possible_regs[0]["register"]:
possible_regs[0]["register"]["mask"] = True
else:
possible_regs[0]['register']['name'] = 'gpr'
elif isa == 'aarch64':
del possible_regs['register']['name']
for key in possible_regs[0]['register']:
parameter[key] = possible_regs[0]['register'][key]
possible_regs[0]["register"]["name"] = "gpr"
elif isa == "aarch64":
del possible_regs["register"]["name"]
for key in possible_regs[0]["register"]:
parameter[key] = possible_regs[0]["register"][key]
parameters.append(parameter)
elif p_type == 'relbr':
parameter['class'] = 'identifier'
elif p_type == "relbr":
parameter["class"] = "identifier"
parameters.append(parameter)
elif p_type == 'agen':
parameter['class'] = 'memory'
parameter['base'] = "*"
parameter['offset'] = "*"
parameter['index'] = "*"
parameter['scale'] = "*"
elif p_type == "agen":
parameter["class"] = "memory"
parameter["base"] = "*"
parameter["offset"] = "*"
parameter["index"] = "*"
parameter["scale"] = "*"
parameters.append(parameter)
else:
raise ValueError("Unknown paramter type {}".format(parameter_tag.attrib))
@@ -113,19 +113,19 @@ def extract_model(tree, arch, skip_mem=True):
mm = MachineModel(isa=isa)
parser = get_parser(isa)
for instruction_tag in tree.findall('.//instruction'):
for instruction_tag in tree.findall(".//instruction"):
ignore = False
mnemonic = instruction_tag.attrib['asm']
iform = instruction_tag.attrib['iform']
mnemonic = instruction_tag.attrib["asm"]
iform = instruction_tag.attrib["iform"]
# skip any mnemonic which contain spaces (e.g., "REX CRC32")
if ' ' in mnemonic:
if " " in mnemonic:
continue
# Extract parameter components
try:
parameters = extract_paramters(instruction_tag, parser, isa)
if isa == 'x86':
if isa == "x86":
parameters.reverse()
except ValueError as e:
print(e, file=sys.stderr)
@@ -136,13 +136,11 @@ def extract_model(tree, arch, skip_mem=True):
if arch_tag is None:
continue
# skip any instructions without port utilization
if not any(['ports' in x.attrib for x in arch_tag.findall('measurement')]):
if not any(["ports" in x.attrib for x in arch_tag.findall("measurement")]):
print("Couldn't find port utilization, skip: ", iform, file=sys.stderr)
continue
# skip if computed and measured TP don't match
if not [x.attrib['TP_ports'] == x.attrib['TP'] for x in arch_tag.findall('measurement')][
0
]:
if not [x.attrib["TP_ports"] == x.attrib["TP"] for x in arch_tag.findall("measurement")][0]:
print(
"Calculated TP from port utilization doesn't match TP, skip: ",
iform,
@@ -151,33 +149,31 @@ def extract_model(tree, arch, skip_mem=True):
continue
# skip if instruction contains memory operand
if skip_mem and any(
[x.attrib['type'] == 'mem' for x in instruction_tag.findall('operand')]
[x.attrib["type"] == "mem" for x in instruction_tag.findall("operand")]
):
print("Contains memory operand, skip: ", iform, file=sys.stderr)
continue
# We collect all measurement and IACA information and compare them later
for measurement_tag in arch_tag.iter('measurement'):
if 'TP_ports' in measurement_tag.attrib:
throughput = measurement_tag.attrib['TP_ports']
for measurement_tag in arch_tag.iter("measurement"):
if "TP_ports" in measurement_tag.attrib:
throughput = measurement_tag.attrib["TP_ports"]
else:
throughput = (
measurement_tag.attrib['TP'] if 'TP' in measurement_tag.attrib else None
measurement_tag.attrib["TP"] if "TP" in measurement_tag.attrib else None
)
uops = (
int(measurement_tag.attrib['uops']) if 'uops' in measurement_tag.attrib else None
)
if 'ports' in measurement_tag.attrib:
uops = int(measurement_tag.attrib["uops"]) if "uops" in measurement_tag.attrib else None
if "ports" in measurement_tag.attrib:
port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib))
latencies = [
int(l_tag.attrib['cycles'])
for l_tag in measurement_tag.iter('latency')
if 'cycles' in l_tag.attrib
int(l_tag.attrib["cycles"])
for l_tag in measurement_tag.iter("latency")
if "cycles" in l_tag.attrib
]
if len(latencies) == 0:
latencies = [
int(l_tag.attrib['max_cycles'])
for l_tag in measurement_tag.iter('latency')
if 'max_cycles' in l_tag.attrib
int(l_tag.attrib["max_cycles"])
for l_tag in measurement_tag.iter("latency")
if "max_cycles" in l_tag.attrib
]
if latencies[1:] != latencies[:-1]:
print(
@@ -193,9 +189,9 @@ def extract_model(tree, arch, skip_mem=True):
# Ordered by IACA version (newest last)
for iaca_tag in sorted(
arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version'])
arch_tag.iter("IACA"), key=lambda i: StrictVersion(i.attrib["version"])
):
if 'ports' in iaca_tag.attrib:
if "ports" in iaca_tag.attrib:
port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib))
# Check if all are equal
@@ -208,26 +204,27 @@ def extract_model(tree, arch, skip_mem=True):
continue
# Adding Intel's 2D and 3D pipelines on Intel µarchs, without Ice Lake:
if arch.upper() in intel_archs and not arch.upper() in ['ICL']:
if any([p['class'] == 'memory' for p in parameters]):
if arch.upper() in intel_archs and not arch.upper() in ["ICL"]:
if any([p["class"] == "memory" for p in parameters]):
# We have a memory parameter, if ports 2 & 3 are present, also add 2D & 3D
# TODO remove port7 on 'hsw' onward and split entries depending on addressing mode
port_23 = False
port_4 = False
for i, pp in enumerate(port_pressure):
if '2' in pp[1] and '3' in pp[1]:
if "2" in pp[1] and "3" in pp[1]:
port_23 = True
if '4' in pp[1]:
if "4" in pp[1]:
port_4 = True
# Add (X, ['2D', '3D']) if load ports (2 & 3) are used, but not the store port (4)
# X = 2 on SNB and IVB IFF used in combination with ymm register, otherwise X = 1
if arch.upper() in ['SNB', 'IVB'] and \
any([p['class'] == 'register' and p['name'] == 'ymm' for p in parameters]):
if arch.upper() in ["SNB", "IVB"] and any(
[p["class"] == "register" and p["name"] == "ymm" for p in parameters]
):
data_port_throughput = 2
else:
data_port_throughput = 1
if port_23 and not port_4:
port_pressure.append((data_port_throughput, ['2D', '3D']))
port_pressure.append((data_port_throughput, ["2D", "3D"]))
# Add missing ports:
for ports in [pp[1] for pp in port_pressure]:
@@ -242,59 +239,55 @@ def extract_model(tree, arch, skip_mem=True):
def rhs_comment(uncommented_string, comment):
max_length = max([len(l) for l in uncommented_string.split('\n')])
max_length = max([len(l) for l in uncommented_string.split("\n")])
commented_string = ""
for l in uncommented_string.split('\n'):
for l in uncommented_string.split("\n"):
commented_string += ("{:<" + str(max_length) + "} # {}\n").format(l, comment)
return commented_string
def architectures(tree):
return set([a.attrib['name'] for a in tree.findall('.//architecture')])
return set([a.attrib["name"] for a in tree.findall(".//architecture")])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('xml', help='path of instructions.xml from http://uops.info')
parser.add_argument("xml", help="path of instructions.xml from http://uops.info")
parser.add_argument(
'arch',
nargs='?',
help='architecture to extract, use IACA abbreviations (e.g., SNB). '
'if not given, all will be extracted and saved to file in CWD.',
"arch",
nargs="?",
help="architecture to extract, use IACA abbreviations (e.g., SNB). "
"if not given, all will be extracted and saved to file in CWD.",
)
parser.add_argument(
'--mem',
dest='skip_mem',
action='store_false',
help='add instruction forms including memory addressing operands, which are '
'skipped by default'
"--mem",
dest="skip_mem",
action="store_false",
help="add instruction forms including memory addressing operands, which are "
"skipped by default",
)
args = parser.parse_args()
basename = os.path.basename(__file__)
tree = ET.parse(args.xml)
print('# Available architectures:', ', '.join(architectures(tree)))
print("# Available architectures:", ", ".join(architectures(tree)))
if args.arch:
print('# Chosen architecture: {}'.format(args.arch))
print("# Chosen architecture: {}".format(args.arch))
model = extract_model(tree, args.arch, args.skip_mem)
if model is not None:
print(
rhs_comment(
model.dump(), "uops.info import"
)
)
print(rhs_comment(model.dump(), "uops.info import"))
else:
for arch in architectures(tree):
print(arch, end='')
print(arch, end="")
model = extract_model(tree, arch.lower(), args.skip_mem)
if model:
model_string = rhs_comment(model.dump(), basename + " " + arch)
with open('{}.yml'.format(arch.lower()), 'w') as f:
with open("{}.yml".format(arch.lower()), "w") as f:
f.write(model_string)
print('.')
print(".")
if __name__ == '__main__':
if __name__ == "__main__":
main()

View File

@@ -30,10 +30,10 @@ def sanity_check(arch: str, verbose=False, internet_check=False, output_file=sys
"""
# load arch machine model
arch_mm = MachineModel(arch=arch)
data = arch_mm['instruction_forms']
data = arch_mm["instruction_forms"]
# load isa machine model
isa = arch_mm.get_ISA()
isa_mm = MachineModel(arch='isa/{}'.format(isa))
isa_mm = MachineModel(arch="isa/{}".format(isa))
num_of_instr = len(data)
# check arch DB entries
@@ -79,17 +79,17 @@ def import_benchmark_output(arch, bench_type, filepath, output=sys.stdout):
:param output: output stream to dump, defaults to sys.stdout
:type output: stream
"""
supported_bench_outputs = ['ibench', 'asmbench']
supported_bench_outputs = ["ibench", "asmbench"]
assert os.path.exists(filepath)
if bench_type not in supported_bench_outputs:
raise ValueError('Benchmark type is not supported.')
with open(filepath, 'r') as f:
raise ValueError("Benchmark type is not supported.")
with open(filepath, "r") as f:
input_data = f.readlines()
db_entries = None
mm = MachineModel(arch)
if bench_type == 'ibench':
if bench_type == "ibench":
db_entries = _get_ibench_output(input_data, mm.get_ISA())
elif bench_type == 'asmbench':
elif bench_type == "asmbench":
db_entries = _get_asmbench_output(input_data, mm.get_ISA())
# write entries to DB
for entry in db_entries:
@@ -122,34 +122,34 @@ def _get_asmbench_output(input_data, isa):
"""
db_entries = {}
for i in range(0, len(input_data), 4):
if input_data[i + 3].strip() != '':
print('asmbench output not in the correct format! Format must be: ', file=sys.stderr)
if input_data[i + 3].strip() != "":
print("asmbench output not in the correct format! Format must be: ", file=sys.stderr)
print(
'-------------\nMNEMONIC[-OP1[_OP2][...]]\nLatency: X cycles\n'
'Throughput: Y cycles\n\n-------------',
"-------------\nMNEMONIC[-OP1[_OP2][...]]\nLatency: X cycles\n"
"Throughput: Y cycles\n\n-------------",
file=sys.stderr,
)
print(
'Entry {} and all further entries won\'t be added.'.format((i / 4) + 1),
"Entry {} and all further entries won't be added.".format((i / 4) + 1),
file=sys.stderr,
)
break
else:
i_form = input_data[i].strip()
mnemonic = i_form.split('-')[0]
operands = i_form.split('-')[1].split('_')
mnemonic = i_form.split("-")[0]
operands = i_form.split("-")[1].split("_")
operands = [_create_db_operand(op, isa) for op in operands]
entry = {
'name': mnemonic,
'operands': operands,
'throughput': _validate_measurement(float(input_data[i + 2].split()[1]), 'tp'),
'latency': _validate_measurement(float(input_data[i + 1].split()[1]), 'lt'),
'port_pressure': None,
"name": mnemonic,
"operands": operands,
"throughput": _validate_measurement(float(input_data[i + 2].split()[1]), "tp"),
"latency": _validate_measurement(float(input_data[i + 1].split()[1]), "lt"),
"port_pressure": None,
}
if not entry['throughput'] or not entry['latency']:
if not entry["throughput"] or not entry["latency"]:
warnings.warn(
'Your measurement for {} looks suspicious'.format(i_form)
+ ' and was not added. Please inspect your benchmark.'
"Your measurement for {} looks suspicious".format(i_form)
+ " and was not added. Please inspect your benchmark."
)
db_entries[i_form] = entry
return db_entries
@@ -159,37 +159,37 @@ def _get_ibench_output(input_data, isa):
"""Parse the standard output of ibench and add instructions to DB."""
db_entries = {}
for line in input_data:
if 'Using frequency' in line or len(line) == 0:
if "Using frequency" in line or len(line) == 0:
continue
instruction = line.split(':')[0]
key = '-'.join(instruction.split('-')[:2])
instruction = line.split(":")[0]
key = "-".join(instruction.split("-")[:2])
if key in db_entries:
# add only TP/LT value
entry = db_entries[key]
else:
mnemonic = instruction.split('-')[0]
operands = instruction.split('-')[1].split('_')
mnemonic = instruction.split("-")[0]
operands = instruction.split("-")[1].split("_")
operands = [_create_db_operand(op, isa) for op in operands]
entry = {
'name': mnemonic,
'operands': operands,
'throughput': None,
'latency': None,
'port_pressure': None,
"name": mnemonic,
"operands": operands,
"throughput": None,
"latency": None,
"port_pressure": None,
}
if 'TP' in instruction:
entry['throughput'] = _validate_measurement(float(line.split()[1]), 'tp')
if not entry['throughput']:
if "TP" in instruction:
entry["throughput"] = _validate_measurement(float(line.split()[1]), "tp")
if not entry["throughput"]:
warnings.warn(
'Your THROUGHPUT measurement for {} looks suspicious'.format(key)
+ ' and was not added. Please inspect your benchmark.'
"Your THROUGHPUT measurement for {} looks suspicious".format(key)
+ " and was not added. Please inspect your benchmark."
)
elif 'LT' in instruction:
entry['latency'] = _validate_measurement(float(line.split()[1]), 'lt')
if not entry['latency']:
elif "LT" in instruction:
entry["latency"] = _validate_measurement(float(line.split()[1]), "lt")
if not entry["latency"]:
warnings.warn(
'Your LATENCY measurement for {} looks suspicious'.format(key)
+ ' and was not added. Please inspect your benchmark.'
"Your LATENCY measurement for {} looks suspicious".format(key)
+ " and was not added. Please inspect your benchmark."
)
db_entries[key] = entry
return db_entries
@@ -200,7 +200,7 @@ def _validate_measurement(measurement, mode):
Check if latency has a maximum deviation of 0.05% and throughput is a reciprocal of a
an integer number.
"""
if mode == 'lt':
if mode == "lt":
if (
math.floor(measurement) * 1.05 >= measurement
or math.ceil(measurement) * 0.95 <= measurement
@@ -208,7 +208,7 @@ def _validate_measurement(measurement, mode):
# Value is probably correct, so round it to the estimated value
return float(round(measurement))
# Check reciprocal only if it is a throughput value
elif mode == 'tp':
elif mode == "tp":
reciprocals = [1 / x for x in range(1, 11)]
for reci in reciprocals:
if reci * 0.95 <= measurement <= reci * 1.05:
@@ -221,56 +221,56 @@ def _validate_measurement(measurement, mode):
def _create_db_operand(operand, isa):
"""Get DB operand by input string and ISA."""
if isa == 'aarch64':
if isa == "aarch64":
return _create_db_operand_aarch64(operand)
elif isa == 'x86':
elif isa == "x86":
return _create_db_operand_x86(operand)
def _create_db_operand_aarch64(operand):
"""Get DB operand for AArch64 by operand string."""
if operand == 'i':
return {'class': 'immediate', 'imd': 'int'}
elif operand in 'wxbhsdq':
return {'class': 'register', 'prefix': operand}
elif operand.startswith('v'):
if operand == "i":
return {"class": "immediate", "imd": "int"}
elif operand in "wxbhsdq":
return {"class": "register", "prefix": operand}
elif operand.startswith("v"):
return {
'class': 'register',
'prefix': 'v',
'shape': operand[1:2] if operand[1:2] != '' else 'd',
"class": "register",
"prefix": "v",
"shape": operand[1:2] if operand[1:2] != "" else "d",
}
elif operand.startswith('m'):
elif operand.startswith("m"):
return {
'class': 'memory',
'base': 'x' if 'b' in operand else None,
'offset': 'imd' if 'o' in operand else None,
'index': 'gpr' if 'i' in operand else None,
'scale': 8 if 's' in operand else 1,
'pre-indexed': True if 'r' in operand else False,
'post-indexed': True if 'p' in operand else False,
"class": "memory",
"base": "x" if "b" in operand else None,
"offset": "imd" if "o" in operand else None,
"index": "gpr" if "i" in operand else None,
"scale": 8 if "s" in operand else 1,
"pre-indexed": True if "r" in operand else False,
"post-indexed": True if "p" in operand else False,
}
else:
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
raise ValueError("Parameter {} is not a valid operand code".format(operand))
def _create_db_operand_x86(operand):
"""Get DB operand for AArch64 by operand string."""
if operand == 'r':
return {'class': 'register', 'name': 'gpr'}
elif operand in 'xyz':
return {'class': 'register', 'name': operand + 'mm'}
elif operand == 'i':
return {'class': 'immediate', 'imd': 'int'}
elif operand.startswith('m'):
if operand == "r":
return {"class": "register", "name": "gpr"}
elif operand in "xyz":
return {"class": "register", "name": operand + "mm"}
elif operand == "i":
return {"class": "immediate", "imd": "int"}
elif operand.startswith("m"):
return {
'class': 'memory',
'base': 'gpr' if 'b' in operand else None,
'offset': 'imd' if 'o' in operand else None,
'index': 'gpr' if 'i' in operand else None,
'scale': 8 if 's' in operand else 1,
"class": "memory",
"base": "gpr" if "b" in operand else None,
"offset": "imd" if "o" in operand else None,
"index": "gpr" if "i" in operand else None,
"scale": 8 if "s" in operand else 1,
}
else:
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
raise ValueError("Parameter {} is not a valid operand code".format(operand))
########################
@@ -286,14 +286,14 @@ def _scrape_from_felixcloutier(mnemonic):
from bs4 import BeautifulSoup
except ImportError:
print(
'Module BeautifulSoup not installed. Fetching instruction form information '
'online requires BeautifulSoup.\nUse \'pip install bs4\' for installation.',
"Module BeautifulSoup not installed. Fetching instruction form information "
"online requires BeautifulSoup.\nUse 'pip install bs4' for installation.",
file=sys.stderr,
)
sys.exit(1)
index = 'https://www.felixcloutier.com/x86/index.html'
base_url = 'https://www.felixcloutier.com/x86/'
index = "https://www.felixcloutier.com/x86/index.html"
base_url = "https://www.felixcloutier.com/x86/"
url = base_url + mnemonic.lower()
suspicious = True
@@ -303,8 +303,8 @@ def _scrape_from_felixcloutier(mnemonic):
r = requests.get(url=url)
if r.status_code == 200:
# Found result
operand_enc = BeautifulSoup(r.text, 'html.parser').find(
'h2', attrs={'id': 'instruction-operand-encoding'}
operand_enc = BeautifulSoup(r.text, "html.parser").find(
"h2", attrs={"id": "instruction-operand-encoding"}
)
if operand_enc:
# operand encoding found, otherwise, no need to mark as suspicous
@@ -312,37 +312,35 @@ def _scrape_from_felixcloutier(mnemonic):
operands = _get_src_dst_from_table(table)
elif r.status_code == 404:
# Check for alternative href
index = BeautifulSoup(requests.get(url=index).text, 'html.parser')
alternatives = [ref for ref in index.findAll('a') if ref.text == mnemonic.upper()]
index = BeautifulSoup(requests.get(url=index).text, "html.parser")
alternatives = [ref for ref in index.findAll("a") if ref.text == mnemonic.upper()]
if len(alternatives) > 0:
# alternative(s) found, take first one
url = base_url + alternatives[0].attrs['href'][2:]
operand_enc = BeautifulSoup(requests.get(url=url).text, 'html.parser').find(
'h2', attrs={'id': 'instruction-operand-encoding'}
url = base_url + alternatives[0].attrs["href"][2:]
operand_enc = BeautifulSoup(requests.get(url=url).text, "html.parser").find(
"h2", attrs={"id": "instruction-operand-encoding"}
)
if operand_enc:
# operand encoding found, otherwise, no need to mark as suspicous
table = (
operand_enc.findNextSibling()
)
table = operand_enc.findNextSibling()
operands = _get_src_dst_from_table(table)
if operands:
# Found src/dst assignment for NUM_OPERANDS
if not any(['r' in x and 'w' in x for x in operands]):
if not any(["r" in x and "w" in x for x in operands]):
suspicious = False
return (suspicious, ' '.join(operands))
return (suspicious, " ".join(operands))
def _get_src_dst_from_table(table, num_operands=2):
"""Prettify bs4 table object to string for user"""
# Parse table
header = [''.join(x.string.lower().split()) for x in table.find('tr').findAll('td')]
data = table.findAll('tr')[1:]
header = ["".join(x.string.lower().split()) for x in table.find("tr").findAll("td")]
data = table.findAll("tr")[1:]
data_dict = OrderedDict()
for i, row in enumerate(data):
data_dict[i] = {}
for j, col in enumerate(row.findAll('td')):
if col.string != 'NA':
for j, col in enumerate(row.findAll("td")):
if col.string != "NA":
data_dict[i][header[j]] = col.string
# Get only the instruction forms with 2 operands
num_ops = [_get_number_of_operands(row) for _, row in data_dict.items()]
@@ -350,12 +348,12 @@ def _get_src_dst_from_table(table, num_operands=2):
row = data_dict[num_ops.index(num_operands)]
reads_writes = []
for i in range(1, num_operands + 1):
m = re.search(r'(\([^\(\)]+\))', row['operand{}'.format(i)])
m = re.search(r"(\([^\(\)]+\))", row["operand{}".format(i)])
if not m:
# no parentheses (probably immediate operand), assume READ
reads_writes.append('(r)')
reads_writes.append("(r)")
continue
reads_writes.append(''.join(m.group(0).split()))
reads_writes.append("".join(m.group(0).split()))
# reverse reads_writes for AT&T syntax
reads_writes.reverse()
return reads_writes
@@ -366,7 +364,7 @@ def _get_number_of_operands(data_dict_row):
"""Return the number of `Operand [X]` attributes in row"""
num = 0
for i in range(1, 5):
if 'operand{}'.format(i) in [''.join(x.split()).lower() for x in data_dict_row]:
if "operand{}".format(i) in ["".join(x.split()).lower() for x in data_dict_row]:
num += 1
return num
@@ -374,12 +372,12 @@ def _get_number_of_operands(data_dict_row):
def _check_sanity_arch_db(arch_mm, isa_mm, internet_check=True):
"""Do sanity check for ArchDB by given ISA."""
# prefixes of instruction forms which we assume to have non-default operands
suspicious_prefixes_x86 = ['vfm', 'fm']
suspicious_prefixes_arm = ['fml', 'ldp', 'stp', 'str']
suspicious_prefixes_x86 = ["vfm", "fm"]
suspicious_prefixes_arm = ["fml", "ldp", "stp", "str"]
# already known to be default-operand instruction forms with 2 operands
if arch_mm.get_ISA().lower() == 'aarch64':
if arch_mm.get_ISA().lower() == "aarch64":
suspicious_prefixes = suspicious_prefixes_arm
if arch_mm.get_ISA().lower() == 'x86':
if arch_mm.get_ISA().lower() == "x86":
suspicious_prefixes = suspicious_prefixes_x86
# returned lists
@@ -391,57 +389,56 @@ def _check_sanity_arch_db(arch_mm, isa_mm, internet_check=True):
duplicate_strings = []
bad_operand = []
for instr_form in arch_mm['instruction_forms']:
for instr_form in arch_mm["instruction_forms"]:
# check value in DB entry
if instr_form['throughput'] is None:
if instr_form["throughput"] is None:
missing_throughput.append(instr_form)
if instr_form['latency'] is None:
if instr_form["latency"] is None:
missing_latency.append(instr_form)
if instr_form['port_pressure'] is None:
if instr_form["port_pressure"] is None:
missing_port_pressure.append(instr_form)
# check entry against ISA DB
for prefix in suspicious_prefixes:
if instr_form['name'].lower().startswith(prefix):
if instr_form["name"].lower().startswith(prefix):
# check if instruction in ISA DB
if isa_mm.get_instruction(instr_form['name'], instr_form['operands']) is None:
if isa_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None:
# if not, mark them as suspicious and print it on the screen
suspicious_instructions.append(instr_form)
# instr forms with less than 3 operands might need an ISA DB entry due to src_reg operands
if (
len(instr_form['operands']) < 3
and len(instr_form['operands']) > 1
and 'mov' not in instr_form['name'].lower()
and not instr_form['name'].lower().startswith('j')
len(instr_form["operands"]) < 3
and len(instr_form["operands"]) > 1
and "mov" not in instr_form["name"].lower()
and not instr_form["name"].lower().startswith("j")
and instr_form not in suspicious_instructions
and isa_mm.get_instruction(instr_form['name'], instr_form['operands']) is None
and isa_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None
):
# validate with data from internet if connected flag is set
if internet_check:
is_susp, info_string = _scrape_from_felixcloutier(instr_form['name'])
is_susp, info_string = _scrape_from_felixcloutier(instr_form["name"])
if is_susp:
instr_form['note'] = info_string
instr_form["note"] = info_string
suspicious_instructions.append(instr_form)
else:
suspicious_instructions.append(instr_form)
# check for duplicates in DB
if arch_mm._check_for_duplicate(instr_form['name'], instr_form['operands']):
if arch_mm._check_for_duplicate(instr_form["name"], instr_form["operands"]):
duplicate_instr_arch.append(instr_form)
# Check operands
for operand in instr_form['operands']:
if operand['class'] == 'register' and not (
'name' in operand or
'prefix' in operand):
for operand in instr_form["operands"]:
if operand["class"] == "register" and not ("name" in operand or "prefix" in operand):
# Missing 'name' key
bad_operand.append(instr_form)
elif operand['class'] == 'memory' and (
'base' not in operand or
'offset' not in operand or
'index' not in operand or
'scale' not in operand):
elif operand["class"] == "memory" and (
"base" not in operand
or "offset" not in operand
or "index" not in operand
or "scale" not in operand
):
# Missing at least one key necessary for memory operands
bad_operand.append(instr_form)
elif operand['class'] == 'immediate' and 'imd' not in operand:
elif operand["class"] == "immediate" and "imd" not in operand:
# Missing 'imd' key
bad_operand.append(instr_form)
# every entry exists twice --> uniquify
@@ -468,12 +465,12 @@ def _check_sanity_isa_db(arch_mm, isa_mm):
duplicate_instr_isa = []
only_in_isa = []
for instr_form in isa_mm['instruction_forms']:
for instr_form in isa_mm["instruction_forms"]:
# check if instr is missing in arch DB
if arch_mm.get_instruction(instr_form['name'], instr_form['operands']) is None:
if arch_mm.get_instruction(instr_form["name"], instr_form["operands"]) is None:
only_in_isa.append(instr_form)
# check for duplicates
if isa_mm._check_for_duplicate(instr_form['name'], instr_form['operands']):
if isa_mm._check_for_duplicate(instr_form["name"], instr_form["operands"]):
duplicate_instr_isa.append(instr_form)
# every entry exists twice --> uniquify
tmp_list = []
@@ -486,82 +483,102 @@ def _check_sanity_isa_db(arch_mm, isa_mm):
return duplicate_instr_isa, only_in_isa
def _get_sanity_report(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa,
bad_operands, verbose=False, colors=False):
def _get_sanity_report(
total,
m_tp,
m_l,
m_pp,
suspic_instr,
dup_arch,
dup_isa,
only_isa,
bad_operands,
verbose=False,
colors=False,
):
"""Get sanity summary report."""
s = ''
s = ""
# non-verbose summary
s += 'SUMMARY\n----------------------\n'
s += '{}% ({}/{}) of instruction forms have no throughput value.\n'.format(
s += "SUMMARY\n----------------------\n"
s += "{}% ({}/{}) of instruction forms have no throughput value.\n".format(
round(100 * len(m_tp) / total), len(m_tp), total
)
s += '{}% ({}/{}) of instruction forms have no latency value.\n'.format(
s += "{}% ({}/{}) of instruction forms have no latency value.\n".format(
round(100 * len(m_l) / total), len(m_l), total
)
s += '{}% ({}/{}) of instruction forms have no port pressure assignment.\n'.format(
s += "{}% ({}/{}) of instruction forms have no port pressure assignment.\n".format(
round(100 * len(m_pp) / total), len(m_pp), total
)
s += '{}% ({}/{}) of instruction forms might miss an ISA DB entry.\n'.format(
s += "{}% ({}/{}) of instruction forms might miss an ISA DB entry.\n".format(
round(100 * len(suspic_instr) / total), len(suspic_instr), total
)
s += '{} duplicate instruction forms in uarch DB.\n'.format(len(dup_arch))
s += '{} duplicate instruction forms in ISA DB.\n'.format(len(dup_isa))
s += "{} duplicate instruction forms in uarch DB.\n".format(len(dup_arch))
s += "{} duplicate instruction forms in ISA DB.\n".format(len(dup_isa))
s += (
'{} instruction forms in ISA DB are not referenced by instruction '.format(len(only_isa))
+ 'forms in uarch DB.\n'
"{} instruction forms in ISA DB are not referenced by instruction ".format(len(only_isa))
+ "forms in uarch DB.\n"
)
s += '{} bad operands found in uarch DB\n'.format(len(bad_operands))
s += '----------------------\n'
s += "{} bad operands found in uarch DB\n".format(len(bad_operands))
s += "----------------------\n"
# verbose version
if verbose:
s += _get_sanity_report_verbose(
total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, bad_operands,
colors=colors
total,
m_tp,
m_l,
m_pp,
suspic_instr,
dup_arch,
dup_isa,
only_isa,
bad_operands,
colors=colors,
)
return s
def _get_sanity_report_verbose(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa,
bad_operands, colors=False):
def _get_sanity_report_verbose(
total, m_tp, m_l, m_pp, suspic_instr, dup_arch, dup_isa, only_isa, bad_operands, colors=False
):
"""Get the verbose part of the sanity report with all missing instruction forms."""
BRIGHT_CYAN = '\033[1;36;1m' if colors else ''
BRIGHT_BLUE = '\033[1;34;1m' if colors else ''
BRIGHT_RED = '\033[1;31;1m' if colors else ''
BRIGHT_MAGENTA = '\033[1;35;1m' if colors else ''
BRIGHT_YELLOW = '\033[1;33;1m' if colors else ''
CYAN = '\033[36m' if colors else ''
YELLOW = '\033[33m' if colors else ''
WHITE = '\033[0m' if colors else ''
BRIGHT_CYAN = "\033[1;36;1m" if colors else ""
BRIGHT_BLUE = "\033[1;34;1m" if colors else ""
BRIGHT_RED = "\033[1;31;1m" if colors else ""
BRIGHT_MAGENTA = "\033[1;35;1m" if colors else ""
BRIGHT_YELLOW = "\033[1;33;1m" if colors else ""
CYAN = "\033[36m" if colors else ""
YELLOW = "\033[33m" if colors else ""
WHITE = "\033[0m" if colors else ""
s = 'Instruction forms without throughput value:\n' if m_tp else ''
for instr_form in sorted(m_tp, key=lambda i: i['name']):
s += '{}{}{}\n'.format(BRIGHT_BLUE, _get_full_instruction_name(instr_form), WHITE)
s += 'Instruction forms without latency value:\n' if m_l else ''
for instr_form in sorted(m_l, key=lambda i: i['name']):
s += '{}{}{}\n'.format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE)
s += 'Instruction forms without port pressure assignment:\n' if m_pp else ''
for instr_form in sorted(m_pp, key=lambda i: i['name']):
s += '{}{}{}\n'.format(BRIGHT_MAGENTA, _get_full_instruction_name(instr_form), WHITE)
s += 'Instruction forms which might miss an ISA DB entry:\n' if suspic_instr else ''
for instr_form in sorted(suspic_instr, key=lambda i: i['name']):
s += '{}{}{}{}\n'.format(
s = "Instruction forms without throughput value:\n" if m_tp else ""
for instr_form in sorted(m_tp, key=lambda i: i["name"]):
s += "{}{}{}\n".format(BRIGHT_BLUE, _get_full_instruction_name(instr_form), WHITE)
s += "Instruction forms without latency value:\n" if m_l else ""
for instr_form in sorted(m_l, key=lambda i: i["name"]):
s += "{}{}{}\n".format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE)
s += "Instruction forms without port pressure assignment:\n" if m_pp else ""
for instr_form in sorted(m_pp, key=lambda i: i["name"]):
s += "{}{}{}\n".format(BRIGHT_MAGENTA, _get_full_instruction_name(instr_form), WHITE)
s += "Instruction forms which might miss an ISA DB entry:\n" if suspic_instr else ""
for instr_form in sorted(suspic_instr, key=lambda i: i["name"]):
s += "{}{}{}{}\n".format(
BRIGHT_CYAN,
_get_full_instruction_name(instr_form),
' -- ' + instr_form['note'] if 'note' in instr_form else '',
" -- " + instr_form["note"] if "note" in instr_form else "",
WHITE,
)
s += 'Duplicate instruction forms in uarch DB:\n' if dup_arch else ''
for instr_form in sorted(dup_arch, key=lambda i: i['name']):
s += '{}{}{}\n'.format(YELLOW, _get_full_instruction_name(instr_form), WHITE)
s += 'Duplicate instruction forms in ISA DB:\n' if dup_isa else ''
for instr_form in sorted(dup_isa, key=lambda i: i['name']):
s += '{}{}{}\n'.format(BRIGHT_YELLOW, _get_full_instruction_name(instr_form), WHITE)
s += 'Instruction forms existing in ISA DB but not in uarch DB:\n' if only_isa else ''
for instr_form in sorted(only_isa, key=lambda i: i['name']):
s += '{}{}{}\n'.format(CYAN, _get_full_instruction_name(instr_form), WHITE)
s += '{} bad operands found in uarch DB:\n'.format(len(bad_operands)) if bad_operands else ''
for instr_form in sorted(bad_operands, key=lambda i: i['name']):
s += '{}{}{}\n'.format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE)
s += "Duplicate instruction forms in uarch DB:\n" if dup_arch else ""
for instr_form in sorted(dup_arch, key=lambda i: i["name"]):
s += "{}{}{}\n".format(YELLOW, _get_full_instruction_name(instr_form), WHITE)
s += "Duplicate instruction forms in ISA DB:\n" if dup_isa else ""
for instr_form in sorted(dup_isa, key=lambda i: i["name"]):
s += "{}{}{}\n".format(BRIGHT_YELLOW, _get_full_instruction_name(instr_form), WHITE)
s += "Instruction forms existing in ISA DB but not in uarch DB:\n" if only_isa else ""
for instr_form in sorted(only_isa, key=lambda i: i["name"]):
s += "{}{}{}\n".format(CYAN, _get_full_instruction_name(instr_form), WHITE)
s += "{} bad operands found in uarch DB:\n".format(len(bad_operands)) if bad_operands else ""
for instr_form in sorted(bad_operands, key=lambda i: i["name"]):
s += "{}{}{}\n".format(BRIGHT_RED, _get_full_instruction_name(instr_form), WHITE)
return s
@@ -573,18 +590,18 @@ def _get_sanity_report_verbose(total, m_tp, m_l, m_pp, suspic_instr, dup_arch, d
def _get_full_instruction_name(instruction_form):
"""Get full instruction form name/identifier string out of given instruction form."""
operands = []
for op in instruction_form['operands']:
for op in instruction_form["operands"]:
op_attrs = [
y + ':' + str(op[y])
for y in list(filter(lambda x: True if x != 'class' else False, op))
y + ":" + str(op[y])
for y in list(filter(lambda x: True if x != "class" else False, op))
]
operands.append('{}({})'.format(op['class'], ','.join(op_attrs)))
return '{} {}'.format(instruction_form['name'], ','.join(operands))
operands.append("{}({})".format(op["class"], ",".join(op_attrs)))
return "{} {}".format(instruction_form["name"], ",".join(operands))
def __represent_none(self, data):
"""Get YAML None representation."""
return self.represent_scalar(u'tag:yaml.org,2002:null', u'~')
return self.represent_scalar(u"tag:yaml.org,2002:null", u"~")
def _create_yaml_object():
@@ -598,17 +615,17 @@ def __dump_data_to_yaml(filepath, data):
"""Dump data to YAML file at given filepath."""
# first add 'normal' meta data in the right order (no ordered dict yet)
meta_data = dict(data)
del meta_data['instruction_forms']
del meta_data['port_model_scheme']
with open(filepath, 'w') as f:
del meta_data["instruction_forms"]
del meta_data["port_model_scheme"]
with open(filepath, "w") as f:
ruamel.yaml.dump(meta_data, f, allow_unicode=True)
with open(filepath, 'a') as f:
with open(filepath, "a") as f:
# now add port model scheme in |-scheme for better readability
ruamel.yaml.dump(
{'port_model_scheme': data['port_model_scheme']},
{"port_model_scheme": data["port_model_scheme"]},
f,
allow_unicode=True,
default_style='|',
default_style="|",
)
# finally, add instruction forms
ruamel.yaml.dump({'instruction_forms': data['instruction_forms']}, f, allow_unicode=True)
ruamel.yaml.dump({"instruction_forms": data["instruction_forms"]}, f, allow_unicode=True)

View File

@@ -9,20 +9,19 @@ from datetime import datetime as dt
from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel
def _get_version(*file_paths):
"""Searches for a version attribute in the given file(s)"""
with io.open(
os.path.join(os.path.dirname(__file__), *file_paths), encoding='utf-8'
) as fp:
with io.open(os.path.join(os.path.dirname(__file__), *file_paths), encoding="utf-8") as fp:
version_file = fp.read()
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
if version_match:
return version_match.group(1)
return '?.?.?'
return "?.?.?"
class Frontend(object):
def __init__(self, filename='', arch=None, path_to_yaml=None):
def __init__(self, filename="", arch=None, path_to_yaml=None):
"""
Constructor method.
@@ -35,9 +34,9 @@ class Frontend(object):
"""
self._filename = filename
if not arch and not path_to_yaml:
raise ValueError('Either arch or path_to_yaml required.')
raise ValueError("Either arch or path_to_yaml required.")
if arch and path_to_yaml:
raise ValueError('Only one of arch and path_to_yaml is allowed.')
raise ValueError("Only one of arch and path_to_yaml is allowed.")
self._arch = arch
if arch:
self._arch = arch.lower()
@@ -54,7 +53,7 @@ class Frontend(object):
:type instruction_form: `dict`
:returns: `True` if comment line, `False` otherwise
"""
return instruction_form['comment'] is not None and instruction_form['instruction'] is None
return instruction_form["comment"] is not None and instruction_form["instruction"] is None
def throughput_analysis(self, kernel, show_lineno=False, show_cmnts=True):
"""
@@ -67,40 +66,40 @@ class Frontend(object):
:param show_cmnts: flag for showing comment-only lines in kernel, defaults to `True`
:type show_cmnts: bool, optional
"""
lineno_filler = ' ' if show_lineno else ''
lineno_filler = " " if show_lineno else ""
port_len = self._get_max_port_len(kernel)
separator = '-' * sum([x + 3 for x in port_len]) + '-'
separator += '--' + len(str(kernel[-1]['line_number'])) * '-' if show_lineno else ''
col_sep = '|'
separator = "-" * sum([x + 3 for x in port_len]) + "-"
separator += "--" + len(str(kernel[-1]["line_number"])) * "-" if show_lineno else ""
col_sep = "|"
sep_list = self._get_separator_list(col_sep)
headline = 'Port pressure in cycles'
headline_str = '{{:^{}}}'.format(len(separator))
headline = "Port pressure in cycles"
headline_str = "{{:^{}}}".format(len(separator))
s = '\n\nThroughput Analysis Report\n--------------------------\n'
s += headline_str.format(headline) + '\n'
s += lineno_filler + self._get_port_number_line(port_len) + '\n'
s += separator + '\n'
s = "\n\nThroughput Analysis Report\n--------------------------\n"
s += headline_str.format(headline) + "\n"
s += lineno_filler + self._get_port_number_line(port_len) + "\n"
s += separator + "\n"
for instruction_form in kernel:
line = '{:4d} {} {} {}'.format(
instruction_form['line_number'],
line = "{:4d} {} {} {}".format(
instruction_form["line_number"],
self._get_port_pressure(
instruction_form['port_pressure'], port_len, separator=sep_list
instruction_form["port_pressure"], port_len, separator=sep_list
),
self._get_flag_symbols(instruction_form['flags'])
if instruction_form['instruction'] is not None
else ' ',
instruction_form['line'].strip().replace('\t', ' '),
self._get_flag_symbols(instruction_form["flags"])
if instruction_form["instruction"] is not None
else " ",
instruction_form["line"].strip().replace("\t", " "),
)
line = line if show_lineno else col_sep + col_sep.join(line.split(col_sep)[1:])
if show_cmnts is False and self._is_comment(instruction_form):
continue
s += line + '\n'
s += '\n'
s += line + "\n"
s += "\n"
tp_sum = ArchSemantics.get_throughput_sum(kernel)
s += lineno_filler + self._get_port_pressure(tp_sum, port_len, separator=' ') + '\n'
s += lineno_filler + self._get_port_pressure(tp_sum, port_len, separator=" ") + "\n"
return s
def latency_analysis(self, cp_kernel, separator='|'):
def latency_analysis(self, cp_kernel, separator="|"):
"""
Build a list-based CP analysis report.
@@ -109,29 +108,29 @@ class Frontend(object):
:separator: separator symbol for the columns, defaults to '|'
:type separator: str, optional
"""
s = '\n\nLatency Analysis Report\n-----------------------\n'
s = "\n\nLatency Analysis Report\n-----------------------\n"
for instruction_form in cp_kernel:
s += (
'{:4d} {} {:4.1f} {}{}{} {}'.format(
instruction_form['line_number'],
"{:4d} {} {:4.1f} {}{}{} {}".format(
instruction_form["line_number"],
separator,
instruction_form['latency_cp'],
instruction_form["latency_cp"],
separator,
'X' if INSTR_FLAGS.LT_UNKWN in instruction_form['flags'] else ' ',
"X" if INSTR_FLAGS.LT_UNKWN in instruction_form["flags"] else " ",
separator,
instruction_form['line'],
instruction_form["line"],
)
) + '\n'
) + "\n"
s += (
'\n{:4} {} {:4.1f}'.format(
' ' * max([len(str(instr_form['line_number'])) for instr_form in cp_kernel]),
' ' * len(separator),
sum([instr_form['latency_cp'] for instr_form in cp_kernel]),
"\n{:4} {} {:4.1f}".format(
" " * max([len(str(instr_form["line_number"])) for instr_form in cp_kernel]),
" " * len(separator),
sum([instr_form["latency_cp"] for instr_form in cp_kernel]),
)
) + '\n'
) + "\n"
return s
def loopcarried_dependencies(self, dep_dict, separator='|'):
def loopcarried_dependencies(self, dep_dict, separator="|"):
"""
Print a list-based LCD analysis to the terminal.
@@ -141,23 +140,31 @@ class Frontend(object):
:type separator: str, optional
"""
s = (
'\n\nLoop-Carried Dependencies Analysis Report\n'
+ '-----------------------------------------\n'
"\n\nLoop-Carried Dependencies Analysis Report\n"
+ "-----------------------------------------\n"
)
# TODO find a way to overcome padding for different tab-lengths
for dep in dep_dict:
s += '{:4d} {} {:4.1f} {} {:36}{} {}\n'.format(
s += "{:4d} {} {:4.1f} {} {:36}{} {}\n".format(
dep,
separator,
sum([instr_form['latency_lcd'] for instr_form in dep_dict[dep]['dependencies']]),
sum([instr_form["latency_lcd"] for instr_form in dep_dict[dep]["dependencies"]]),
separator,
dep_dict[dep]['root']['line'].strip(),
dep_dict[dep]["root"]["line"].strip(),
separator,
[node['line_number'] for node in dep_dict[dep]['dependencies']],
[node["line_number"] for node in dep_dict[dep]["dependencies"]],
)
return s
def full_analysis(self, kernel, kernel_dg: KernelDG, ignore_unknown=False, arch_warning=False, length_warning=False, verbose=False):
def full_analysis(
self,
kernel,
kernel_dg: KernelDG,
ignore_unknown=False,
arch_warning=False,
length_warning=False,
verbose=False,
):
"""
Build the full analysis report including header, the symbol map, the combined TP/CP/LCD
view and the list based LCD view.
@@ -208,71 +215,71 @@ class Frontend(object):
:param show_cmnts: flag for showing comment-only lines in kernel, defaults to `True`
:type show_cmnts: bool, optional
"""
s = '\n\nCombined Analysis Report\n------------------------\n'
lineno_filler = ' '
s = "\n\nCombined Analysis Report\n------------------------\n"
lineno_filler = " "
port_len = self._get_max_port_len(kernel)
# Separator for ports
separator = '-' * sum([x + 3 for x in port_len]) + '-'
separator = "-" * sum([x + 3 for x in port_len]) + "-"
# ... for line numbers
separator += '--' + len(str(kernel[-1]['line_number'])) * '-'
col_sep = '|'
separator += "--" + len(str(kernel[-1]["line_number"])) * "-"
col_sep = "|"
# for LCD/CP column
separator += '-' * (2 * 6 + len(col_sep)) + '-' * len(col_sep)
separator += "-" * (2 * 6 + len(col_sep)) + "-" * len(col_sep)
sep_list = self._get_separator_list(col_sep)
headline = 'Port pressure in cycles'
headline_str = '{{:^{}}}'.format(len(separator))
headline = "Port pressure in cycles"
headline_str = "{{:^{}}}".format(len(separator))
# Prepare CP/LCD variable
cp_lines = [x['line_number'] for x in cp_kernel]
cp_lines = [x["line_number"] for x in cp_kernel]
sums = {}
for dep in dep_dict:
sums[dep] = sum(
[instr_form['latency_lcd'] for instr_form in dep_dict[dep]['dependencies']]
[instr_form["latency_lcd"] for instr_form in dep_dict[dep]["dependencies"]]
)
lcd_sum = max(sums.values()) if len(sums) > 0 else 0.0
lcd_lines = []
if len(dep_dict) > 0:
longest_lcd = [line_no for line_no in sums if sums[line_no] == lcd_sum][0]
lcd_lines = [d['line_number'] for d in dep_dict[longest_lcd]['dependencies']]
lcd_lines = [d["line_number"] for d in dep_dict[longest_lcd]["dependencies"]]
s += headline_str.format(headline) + '\n'
s += headline_str.format(headline) + "\n"
s += (
(
lineno_filler
+ self._get_port_number_line(port_len, separator=col_sep)
+ '{}{:^6}{}{:^6}{}'.format(col_sep, 'CP', col_sep, 'LCD', col_sep)
+ "{}{:^6}{}{:^6}{}".format(col_sep, "CP", col_sep, "LCD", col_sep)
)
+ '\n'
+ "\n"
+ separator
+ '\n'
+ "\n"
)
for instruction_form in kernel:
if show_cmnts is False and self._is_comment(instruction_form):
continue
line_number = instruction_form['line_number']
used_ports = [list(uops[1]) for uops in instruction_form['port_uops']]
line_number = instruction_form["line_number"]
used_ports = [list(uops[1]) for uops in instruction_form["port_uops"]]
used_ports = list(set([p for uops_ports in used_ports for p in uops_ports]))
s += '{:4d} {}{} {} {}\n'.format(
s += "{:4d} {}{} {} {}\n".format(
line_number,
self._get_port_pressure(
instruction_form['port_pressure'], port_len, used_ports, sep_list
instruction_form["port_pressure"], port_len, used_ports, sep_list
),
self._get_lcd_cp_ports(
instruction_form['line_number'],
instruction_form["line_number"],
cp_kernel if line_number in cp_lines else None,
dep_dict[longest_lcd] if line_number in lcd_lines else None,
),
self._get_flag_symbols(instruction_form['flags'])
if instruction_form['instruction'] is not None
else ' ',
instruction_form['line'].strip().replace('\t', ' '),
self._get_flag_symbols(instruction_form["flags"])
if instruction_form["instruction"] is not None
else " ",
instruction_form["line"].strip().replace("\t", " "),
)
s += '\n'
s += "\n"
# check for unknown instructions and throw warning if called without --ignore-unknown
if not ignore_unknown and INSTR_FLAGS.TP_UNKWN in [
flag for instr in kernel for flag in instr['flags']
flag for instr in kernel for flag in instr["flags"]
]:
num_missing = len(
[instr['flags'] for instr in kernel if INSTR_FLAGS.TP_UNKWN in instr['flags']]
[instr["flags"] for instr in kernel if INSTR_FLAGS.TP_UNKWN in instr["flags"]]
)
s += self._missing_instruction_error(num_missing)
else:
@@ -280,12 +287,12 @@ class Frontend(object):
tp_sum = ArchSemantics.get_throughput_sum(kernel)
# if ALL instructions are unknown, take a line of 0s
if not tp_sum:
tp_sum = kernel[0]['port_pressure']
cp_sum = sum([x['latency_cp'] for x in cp_kernel])
tp_sum = kernel[0]["port_pressure"]
cp_sum = sum([x["latency_cp"] for x in cp_kernel])
s += (
lineno_filler
+ self._get_port_pressure(tp_sum, port_len, separator=' ')
+ ' {:^6} {:^6}\n'.format(cp_sum, lcd_sum)
+ self._get_port_pressure(tp_sum, port_len, separator=" ")
+ " {:^6} {:^6}\n".format(cp_sum, lcd_sum)
)
return s
@@ -296,43 +303,42 @@ class Frontend(object):
def _missing_instruction_error(self, amount):
"""Returns the warning for if any instruction form in the analysis is missing."""
s = (
'------------------ WARNING: The performance data for {} instructions is missing.'
'------------------\n'
' No final analysis is given. If you want to ignore this\n'
' warning and run the analysis anyway, start osaca with\n'
' --ignore-unknown flag.\n'
'--------------------------------------------------------------------------------'
'----------------{}\n'
).format(amount, '-' * len(str(amount)))
"------------------ WARNING: The performance data for {} instructions is missing."
"------------------\n"
" No final analysis is given. If you want to ignore this\n"
" warning and run the analysis anyway, start osaca with\n"
" --ignore-unknown flag.\n"
"--------------------------------------------------------------------------------"
"----------------{}\n"
).format(amount, "-" * len(str(amount)))
return s
def _user_warnings(self, arch_warning, length_warning):
"""Returns warning texts for giving the user more insight in what he is doing."""
arch_text = (
'WARNING: No micro-architecture was specified and a default uarch was used.\n'
' Specify the uarch with --arch. See --help for more information.\n'
"WARNING: No micro-architecture was specified and a default uarch was used.\n"
" Specify the uarch with --arch. See --help for more information.\n"
)
length_text = (
'WARNING: You are analyzing a large amount of instruction forms. Analysis '
'across loops/block boundaries often do not make much sense.\n'
' Specify the kernel length with --length. See --help for more '
'information.\n'
' If this is intentional, you can safely ignore this message.\n'
"WARNING: You are analyzing a large amount of instruction forms. Analysis "
"across loops/block boundaries often do not make much sense.\n"
" Specify the kernel length with --length. See --help for more "
"information.\n"
" If this is intentional, you can safely ignore this message.\n"
)
warnings = ''
warnings += arch_text if arch_warning else ''
warnings += length_text if length_warning else ''
warnings += '\n'
warnings = ""
warnings += arch_text if arch_warning else ""
warnings += length_text if length_warning else ""
warnings += "\n"
return warnings
def _get_separator_list(self, separator, separator_2=' '):
def _get_separator_list(self, separator, separator_2=" "):
"""Creates column view for seperators in the TP/combined view."""
separator_list = []
for i in range(len(self._machine_model.get_ports()) - 1):
match_1 = re.search(r'\d+', self._machine_model.get_ports()[i])
match_2 = re.search(r'\d+', self._machine_model.get_ports()[i + 1])
match_1 = re.search(r"\d+", self._machine_model.get_ports()[i])
match_2 = re.search(r"\d+", self._machine_model.get_ports()[i + 1])
if match_1 is not None and match_2 is not None and match_1.group() == match_2.group():
separator_list.append(separator_2)
else:
@@ -342,92 +348,92 @@ class Frontend(object):
def _get_flag_symbols(self, flag_obj):
"""Returns flags for a flag object of an instruction"""
string_result = ''
string_result += '*' if INSTR_FLAGS.NOT_BOUND in flag_obj else ''
string_result += 'X' if INSTR_FLAGS.TP_UNKWN in flag_obj else ''
string_result += 'P' if INSTR_FLAGS.HIDDEN_LD in flag_obj else ''
string_result = ""
string_result += "*" if INSTR_FLAGS.NOT_BOUND in flag_obj else ""
string_result += "X" if INSTR_FLAGS.TP_UNKWN in flag_obj else ""
string_result += "P" if INSTR_FLAGS.HIDDEN_LD in flag_obj else ""
# TODO add other flags
string_result += ' ' if len(string_result) == 0 else ''
string_result += " " if len(string_result) == 0 else ""
return string_result
def _get_port_pressure(self, ports, port_len, used_ports=[], separator='|'):
def _get_port_pressure(self, ports, port_len, used_ports=[], separator="|"):
"""Returns line of port pressure for an instruction."""
if not isinstance(separator, list):
separator = [separator for x in ports]
string_result = '{} '.format(separator[-1])
string_result = "{} ".format(separator[-1])
for i in range(len(ports)):
if float(ports[i]) == 0.0 and self._machine_model.get_ports()[i] not in used_ports:
string_result += port_len[i] * ' ' + ' {} '.format(separator[i])
string_result += port_len[i] * " " + " {} ".format(separator[i])
continue
left_len = len(str(float(ports[i])).split('.')[0])
substr = '{:' + str(left_len) + '.' + str(max(port_len[i] - left_len - 1, 0)) + 'f}'
left_len = len(str(float(ports[i])).split(".")[0])
substr = "{:" + str(left_len) + "." + str(max(port_len[i] - left_len - 1, 0)) + "f}"
substr = substr.format(ports[i])
string_result += (
substr + ' {} '.format(separator[i])
if '.' in substr
else '{:.1f}{} '.format(ports[i], separator[i])
substr + " {} ".format(separator[i])
if "." in substr
else "{:.1f}{} ".format(ports[i], separator[i])
)
return string_result[:-1]
def _get_node_by_lineno(self, lineno, kernel):
"""Returns instruction form from kernel by its line number."""
nodes = [instr for instr in kernel if instr['line_number'] == lineno]
nodes = [instr for instr in kernel if instr["line_number"] == lineno]
return nodes[0] if len(nodes) > 0 else None
def _get_lcd_cp_ports(self, line_number, cp_dg, dependency, separator='|'):
def _get_lcd_cp_ports(self, line_number, cp_dg, dependency, separator="|"):
"""Returns the CP and LCD line for one instruction."""
lat_cp = lat_lcd = ''
lat_cp = lat_lcd = ""
if cp_dg:
lat_cp = float(self._get_node_by_lineno(line_number, cp_dg)['latency_cp'])
lat_cp = float(self._get_node_by_lineno(line_number, cp_dg)["latency_cp"])
if dependency:
lat_lcd = float(
self._get_node_by_lineno(line_number, dependency['dependencies'])['latency_lcd']
self._get_node_by_lineno(line_number, dependency["dependencies"])["latency_lcd"]
)
return '{} {:>4} {} {:>4} {}'.format(separator, lat_cp, separator, lat_lcd, separator)
return "{} {:>4} {} {:>4} {}".format(separator, lat_cp, separator, lat_lcd, separator)
def _get_max_port_len(self, kernel):
"""Returns the maximal length needed to print all throughputs of the kernel."""
port_len = [4 for x in self._machine_model.get_ports()]
for instruction_form in kernel:
for i, port in enumerate(instruction_form['port_pressure']):
if len('{:.2f}'.format(port)) > port_len[i]:
port_len[i] = len('{:.2f}'.format(port))
for i, port in enumerate(instruction_form["port_pressure"]):
if len("{:.2f}".format(port)) > port_len[i]:
port_len[i] = len("{:.2f}".format(port))
return port_len
def _get_port_number_line(self, port_len, separator='|'):
def _get_port_number_line(self, port_len, separator="|"):
"""Returns column view of port identificators of machine_model."""
string_result = separator
separator_list = self._get_separator_list(separator, '-')
separator_list = self._get_separator_list(separator, "-")
for i, length in enumerate(port_len):
substr = '{:^' + str(length + 2) + 's}'
substr = "{:^" + str(length + 2) + "s}"
string_result += substr.format(self._machine_model.get_ports()[i]) + separator_list[i]
return string_result
def _header_report(self):
"""Prints header information"""
version = _get_version('__init__.py')
version = _get_version("__init__.py")
adjust = 20
header = ''
header += 'Open Source Architecture Code Analyzer (OSACA) - {}\n'.format(version)
header += 'Analyzed file:'.ljust(adjust) + '{}\n'.format(self._filename)
header += 'Architecture:'.ljust(adjust) + '{}\n'.format(self._arch.upper())
header += 'Timestamp:'.ljust(adjust) + '{}\n'.format(
dt.utcnow().strftime('%Y-%m-%d %H:%M:%S')
header = ""
header += "Open Source Architecture Code Analyzer (OSACA) - {}\n".format(version)
header += "Analyzed file:".ljust(adjust) + "{}\n".format(self._filename)
header += "Architecture:".ljust(adjust) + "{}\n".format(self._arch.upper())
header += "Timestamp:".ljust(adjust) + "{}\n".format(
dt.utcnow().strftime("%Y-%m-%d %H:%M:%S")
)
return header + '\n'
return header + "\n"
def _symbol_map(self):
"""Prints instruction flag map."""
symbol_dict = {
INSTR_FLAGS.NOT_BOUND: 'Instruction micro-ops not bound to a port',
INSTR_FLAGS.TP_UNKWN: 'No throughput/latency information for this instruction in '
+ 'data file',
INSTR_FLAGS.HIDDEN_LD: 'Throughput of LOAD operation can be hidden behind a past '
+ 'or future STORE instruction',
INSTR_FLAGS.NOT_BOUND: "Instruction micro-ops not bound to a port",
INSTR_FLAGS.TP_UNKWN: "No throughput/latency information for this instruction in "
+ "data file",
INSTR_FLAGS.HIDDEN_LD: "Throughput of LOAD operation can be hidden behind a past "
+ "or future STORE instruction",
}
symbol_map = ''
symbol_map = ""
for flag in sorted(symbol_dict.keys()):
symbol_map += ' {} - {}\n'.format(self._get_flag_symbols([flag]), symbol_dict[flag])
symbol_map += " {} - {}\n".format(self._get_flag_symbols([flag]), symbol_dict[flag])
return symbol_map
def _port_binding_summary(self):

View File

@@ -10,27 +10,26 @@ import traceback
from osaca.db_interface import import_benchmark_output, sanity_check
from osaca.frontend import Frontend
from osaca.parser import BaseParser, ParserAArch64, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG,
MachineModel, reduce_to_section)
from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section
SUPPORTED_ARCHS = [
'SNB',
'IVB',
'HSW',
'BDW',
'SKX',
'CSX',
'ICL',
'ZEN1',
'ZEN2',
'TX2',
'N1',
'A64FX',
"SNB",
"IVB",
"HSW",
"BDW",
"SKX",
"CSX",
"ICL",
"ZEN1",
"ZEN2",
"TX2",
"N1",
"A64FX",
]
DEFAULT_ARCHS = {
'aarch64': 'A64FX',
'x86': 'SKX',
"aarch64": "A64FX",
"x86": "SKX",
}
@@ -50,7 +49,7 @@ def __find_version(*file_paths):
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
if version_match:
return version_match.group(1)
raise RuntimeError('Unable to find version string.')
raise RuntimeError("Unable to find version string.")
def get_version():
@@ -59,7 +58,7 @@ def get_version():
:returns: str -- the version string.
"""
return __find_version('__init__.py')
return __find_version("__init__.py")
def create_parser(parser=None):
@@ -73,91 +72,92 @@ def create_parser(parser=None):
# Create parser
if not parser:
parser = argparse.ArgumentParser(
description='Analyzes a marked innermost loop snippet for a given architecture type.',
epilog='For help, examples, documentation and bug reports go to:\nhttps://github.com'
'/RRZE-HPC/OSACA/ | License: AGPLv3',
description="Analyzes a marked innermost loop snippet for a given architecture type.",
epilog="For help, examples, documentation and bug reports go to:\nhttps://github.com"
"/RRZE-HPC/OSACA/ | License: AGPLv3",
)
# Add arguments
parser.add_argument(
'-V', '--version', action='version', version='%(prog)s ' + __find_version('__init__.py')
"-V", "--version", action="version", version="%(prog)s " + __find_version("__init__.py")
)
parser.add_argument(
'--arch',
"--arch",
type=str,
help='Define architecture (SNB, IVB, HSW, BDW, SKX, CSX, ICL, ZEN1, ZEN2, TX2, N1, '
'A64FX). If no architecture is given, OSACA assumes a default uarch for x86/AArch64.',
help="Define architecture (SNB, IVB, HSW, BDW, SKX, CSX, ICL, ZEN1, ZEN2, TX2, N1, "
"A64FX). If no architecture is given, OSACA assumes a default uarch for x86/AArch64.",
)
parser.add_argument(
'--fixed',
action='store_true',
help='Run the throughput analysis with fixed probabilities for all suitable ports per '
'instruction. Otherwise, OSACA will print the optimal port utilization for the kernel.',
"--fixed",
action="store_true",
help="Run the throughput analysis with fixed probabilities for all suitable ports per "
"instruction. Otherwise, OSACA will print the optimal port utilization for the kernel.",
)
parser.add_argument(
'--lines',
"--lines",
type=str,
help='Define lines that should be included in the analysis. This option overwrites any'
' range defined by markers in the assembly. Add either single lines or ranges defined by'
help="Define lines that should be included in the analysis. This option overwrites any"
" range defined by markers in the assembly. Add either single lines or ranges defined by"
' "-" or ":", each entry separated by commas, e.g.: --lines 1,2,8-18,20:24',
)
parser.add_argument(
'--db-check',
dest='check_db',
action='store_true',
"--db-check",
dest="check_db",
action="store_true",
help='Run a sanity check on the by "--arch" specified database. The output depends '
'on the verbosity level.',
"on the verbosity level.",
)
parser.add_argument(
'--online',
dest='internet_check',
action='store_true',
help='Run sanity check with online DB validation (currently felixcloutier) to see the '
'src/dst distribution of the operands. Can be only used in combination with --db-check.',
"--online",
dest="internet_check",
action="store_true",
help="Run sanity check with online DB validation (currently felixcloutier) to see the "
"src/dst distribution of the operands. Can be only used in combination with --db-check.",
)
parser.add_argument(
'--import',
metavar='MICROBENCH',
dest='import_data',
"--import",
metavar="MICROBENCH",
dest="import_data",
type=str,
default=argparse.SUPPRESS,
help='Import a given microbenchmark output file into the corresponding architecture '
help="Import a given microbenchmark output file into the corresponding architecture "
'instruction database. Define the type of microbenchmark either as "ibench" or '
'"asmbench".',
)
parser.add_argument(
'--insert-marker',
dest='insert_marker',
action='store_true',
help='Try to find assembly block containing the loop to analyse and insert byte '
'marker by using Kerncraft.',
"--insert-marker",
dest="insert_marker",
action="store_true",
help="Try to find assembly block containing the loop to analyse and insert byte "
"marker by using Kerncraft.",
)
parser.add_argument(
'--export-graph',
metavar='EXPORT_PATH',
dest='dotpath',
"--export-graph",
metavar="EXPORT_PATH",
dest="dotpath",
default=None,
type=str,
help='Output path for .dot file export. If "." is given, the file will be stored as '
'"./osaca_dg.dot"',
)
parser.add_argument(
'--ignore-unknown',
dest='ignore_unknown',
action='store_true',
help='Ignore if instructions cannot be found in the data file and print analysis anyway.',
"--ignore-unknown",
dest="ignore_unknown",
action="store_true",
help="Ignore if instructions cannot be found in the data file and print analysis anyway.",
)
parser.add_argument(
'--verbose', '-v', action='count', default=0, help='Increases verbosity level.'
"--verbose", "-v", action="count", default=0, help="Increases verbosity level."
)
parser.add_argument(
'--out', '-o',
"--out",
"-o",
default=sys.stdout,
type=argparse.FileType('w'),
help='Write analysis to this file (default to stdout).'
type=argparse.FileType("w"),
help="Write analysis to this file (default to stdout).",
)
parser.add_argument(
'file', type=argparse.FileType('r'), help='Path to object (ASM or instruction file).'
"file", type=argparse.FileType("r"), help="Path to object (ASM or instruction file)."
)
return parser
@@ -170,24 +170,24 @@ def check_arguments(args, parser):
:param args: arguments given from :class:`~argparse.ArgumentParser` after parsing
:param parser: :class:`~argparse.ArgumentParser` object
"""
supported_import_files = ['ibench', 'asmbench']
supported_import_files = ["ibench", "asmbench"]
if args.arch is None and (args.check_db or 'import_data' in args):
if args.arch is None and (args.check_db or "import_data" in args):
parser.error(
'DB check and data import cannot work with a default microarchitecture. '
'Please see --help for all valid architecture codes.'
"DB check and data import cannot work with a default microarchitecture. "
"Please see --help for all valid architecture codes."
)
elif args.arch is not None and args.arch.upper() not in SUPPORTED_ARCHS:
parser.error(
'Microarchitecture not supported. Please see --help for all valid architecture codes.'
"Microarchitecture not supported. Please see --help for all valid architecture codes."
)
if 'import_data' in args and args.import_data not in supported_import_files:
if "import_data" in args and args.import_data not in supported_import_files:
parser.error(
'Microbenchmark not supported for data import. Please see --help for all valid '
'microbenchmark codes.'
"Microbenchmark not supported for data import. Please see --help for all valid "
"microbenchmark codes."
)
if args.internet_check and not args.check_db:
parser.error('--online requires --check-db')
parser.error("--online requires --check-db")
def import_data(benchmark_type, arch, filepath, output_file=sys.stdout):
@@ -203,12 +203,12 @@ def import_data(benchmark_type, arch, filepath, output_file=sys.stdout):
:param output_file: output stream specifying where to write output, defaults to :class:`sys.stdout`
:type output_file: stream, optional
"""
if benchmark_type.lower() == 'ibench':
import_benchmark_output(arch, 'ibench', filepath, output=output_file)
elif benchmark_type.lower() == 'asmbench':
import_benchmark_output(arch, 'asmbench', filepath, output=output_file)
if benchmark_type.lower() == "ibench":
import_benchmark_output(arch, "ibench", filepath, output=output_file)
elif benchmark_type.lower() == "asmbench":
import_benchmark_output(arch, "asmbench", filepath, output=output_file)
else:
raise NotImplementedError('This benchmark input variant is not supported.')
raise NotImplementedError("This benchmark input variant is not supported.")
def insert_byte_marker(args):
@@ -221,9 +221,9 @@ def insert_byte_marker(args):
from kerncraft.incore_model import asm_instrumentation
except ImportError:
print(
'Module kerncraft not installed. Use \'pip install --user '
'kerncraft\' for installation.\nFor more information see '
'https://github.com/RRZE-HPC/kerncraft',
"Module kerncraft not installed. Use 'pip install --user "
"kerncraft' for installation.\nFor more information see "
"https://github.com/RRZE-HPC/kerncraft",
file=sys.stderr,
)
sys.exit(1)
@@ -234,14 +234,14 @@ def insert_byte_marker(args):
asm_instrumentation(
input_file=unmarked_assembly,
output_file=marked_assembly,
block_selection='manual',
pointer_increment='auto_with_manual_fallback',
block_selection="manual",
pointer_increment="auto_with_manual_fallback",
isa=MachineModel.get_isa_for_arch(args.arch),
)
marked_assembly.seek(0)
assembly = marked_assembly.read()
with open(args.file.name, 'w') as f:
with open(args.file.name, "w") as f:
f.write(assembly)
@@ -272,7 +272,11 @@ def inspect(args, output_file=sys.stdout):
# probably the wrong parser based on heuristic
if args.arch is None:
# change ISA and try again
arch = DEFAULT_ARCHS['x86'] if BaseParser.detect_ISA(code) == 'aarch64' else DEFAULT_ARCHS['aarch64']
arch = (
DEFAULT_ARCHS["x86"]
if BaseParser.detect_ISA(code) == "aarch64"
else DEFAULT_ARCHS["aarch64"]
)
isa = MachineModel.get_isa_for_arch(arch)
parser = get_asm_parser(arch)
parsed_code = parser.parse_file(code)
@@ -282,12 +286,14 @@ def inspect(args, output_file=sys.stdout):
# Reduce to marked kernel or chosen section and add semantics
if args.lines:
line_range = get_line_range(args.lines)
kernel = [line for line in parsed_code if line['line_number'] in line_range]
kernel = [line for line in parsed_code if line["line_number"] in line_range]
print_length_warning = False
else:
kernel = reduce_to_section(parsed_code, isa)
# Print warning if kernel has no markers and is larger than threshold (100)
print_length_warning = True if len(kernel) == len(parsed_code) and len(kernel) > 100 else False
print_length_warning = (
True if len(kernel) == len(parsed_code) and len(kernel) > 100 else False
)
machine_model = MachineModel(arch=arch)
semantics = ArchSemantics(machine_model)
semantics.add_semantics(kernel)
@@ -298,7 +304,7 @@ def inspect(args, output_file=sys.stdout):
# Create DiGrahps
kernel_graph = KernelDG(kernel, parser, machine_model)
if args.dotpath is not None:
kernel_graph.export_graph(args.dotpath if args.dotpath != '.' else None)
kernel_graph.export_graph(args.dotpath if args.dotpath != "." else None)
# Print analysis
frontend = Frontend(args.file.name, arch=arch)
print(
@@ -308,7 +314,7 @@ def inspect(args, output_file=sys.stdout):
ignore_unknown=ignore_unknown,
arch_warning=print_arch_warning,
length_warning=print_length_warning,
verbose=verbose
verbose=verbose,
),
file=output_file,
)
@@ -328,7 +334,7 @@ def run(args, output_file=sys.stdout):
sanity_check(
args.arch, verbose=verbose, internet_check=args.internet_check, output_file=output_file
)
elif 'import_data' in args:
elif "import_data" in args:
# Import microbench output file into DB
import_data(args.import_data, args.arch, args.file.name, output_file=output_file)
elif args.insert_marker:
@@ -348,9 +354,9 @@ def get_asm_parser(arch) -> BaseParser:
:returns: :class:`~osaca.parser.BaseParser` object
"""
isa = MachineModel.get_isa_for_arch(arch)
if isa == 'x86':
if isa == "x86":
return ParserX86ATT()
elif isa == 'aarch64':
elif isa == "aarch64":
return ParserAArch64()
@@ -359,26 +365,28 @@ def get_unmatched_instruction_ratio(kernel):
unmatched_counter = 0
for instruction in kernel:
if (
INSTR_FLAGS.TP_UNKWN in instruction['flags']
and INSTR_FLAGS.LT_UNKWN in instruction['flags']
INSTR_FLAGS.TP_UNKWN in instruction["flags"]
and INSTR_FLAGS.LT_UNKWN in instruction["flags"]
):
unmatched_counter += 1
return unmatched_counter / len(kernel)
def get_line_range(line_str):
line_str = line_str.replace(':', '-')
lines = line_str.split(',')
line_str = line_str.replace(":", "-")
lines = line_str.split(",")
lines_int = []
for l in lines:
if '-' in l:
start = int(l.split('-')[0])
end = int(l.split('-')[1])
if "-" in l:
start = int(l.split("-")[0])
end = int(l.split("-")[1])
rnge = list(range(start, end + 1))
lines_int += rnge
else:
lines_int.append(int(l))
return lines_int
def main():
"""Initialize and run command line interface."""
parser = create_parser()
@@ -387,5 +395,5 @@ def main():
run(args, output_file=args.out)
if __name__ == '__main__':
if __name__ == "__main__":
main()

View File

@@ -8,12 +8,13 @@ from .base_parser import BaseParser
from .parser_x86att import ParserX86ATT
from .parser_AArch64 import ParserAArch64
__all__ = ['AttrDict', 'BaseParser', 'ParserX86ATT', 'ParserAArch64', 'get_parser']
__all__ = ["AttrDict", "BaseParser", "ParserX86ATT", "ParserAArch64", "get_parser"]
def get_parser(isa):
if isa.lower() == 'x86':
if isa.lower() == "x86":
return ParserX86ATT()
elif isa.lower() == 'aarch64':
elif isa.lower() == "aarch64":
return ParserAArch64()
else:
raise ValueError("Unknown ISA {!r}.".format(isa))

View File

@@ -3,18 +3,19 @@
import operator
import re
class BaseParser(object):
# Identifiers for operand types
COMMENT_ID = 'comment'
DIRECTIVE_ID = 'directive'
IMMEDIATE_ID = 'immediate'
LABEL_ID = 'label'
IDENTIFIER_ID = 'identifier'
MEMORY_ID = 'memory'
REGISTER_ID = 'register'
SEGMENT_EXT_ID = 'segment_extension'
INSTRUCTION_ID = 'instruction'
OPERANDS_ID = 'operands'
COMMENT_ID = "comment"
DIRECTIVE_ID = "directive"
IMMEDIATE_ID = "immediate"
LABEL_ID = "label"
IDENTIFIER_ID = "identifier"
MEMORY_ID = "memory"
REGISTER_ID = "register"
SEGMENT_EXT_ID = "segment_extension"
INSTRUCTION_ID = "instruction"
OPERANDS_ID = "operands"
_parser_constructed = False
def __init__(self):
@@ -27,15 +28,15 @@ class BaseParser(object):
"""Detect the ISA of the assembly based on the used registers and return the ISA code."""
# Check for the amount of registers in the code to determine the ISA
# 1) Check for xmm, ymm, zmm, rax, rbx, rcx, and rdx registers in x86
heuristics_x86ATT = [r'%[xyz]mm[0-9]', r'%[er][abcd]x[0-9]']
heuristics_x86ATT = [r"%[xyz]mm[0-9]", r"%[er][abcd]x[0-9]"]
# 2) check for v and z vector registers and x/w general-purpose registers
heuristics_aarch64 = [r'[vz][0-9][0-9]?\.[0-9][0-9]?[bhsd]', r'[wx][0-9]']
matches = {'x86': 0, 'aarch64': 0}
heuristics_aarch64 = [r"[vz][0-9][0-9]?\.[0-9][0-9]?[bhsd]", r"[wx][0-9]"]
matches = {"x86": 0, "aarch64": 0}
for h in heuristics_x86ATT:
matches['x86'] += len(re.findall(h, file_content))
matches["x86"] += len(re.findall(h, file_content))
for h in heuristics_aarch64:
matches['aarch64'] += len(re.findall(h, file_content))
matches["aarch64"] += len(re.findall(h, file_content))
return max(matches.items(), key=operator.itemgetter(1))[0]
@@ -50,9 +51,9 @@ class BaseParser(object):
"""
# Create instruction form list
asm_instructions = []
lines = file_content.split('\n')
lines = file_content.split("\n")
for i, line in enumerate(lines):
if line.strip() == '':
if line.strip() == "":
continue
asm_instructions.append(self.parse_line(line, i + 1 + start_line))
return asm_instructions

View File

@@ -17,53 +17,56 @@ class ParserAArch64(BaseParser):
def __init__(self):
super().__init__()
self.isa = 'aarch64'
self.isa = "aarch64"
def construct_parser(self):
"""Create parser for ARM AArch64 ISA."""
# Comment
symbol_comment = '//'
symbol_comment = "//"
self.comment = pp.Literal(symbol_comment) + pp.Group(
pp.ZeroOrMore(pp.Word(pp.printables))
).setResultsName(self.COMMENT_ID)
# Define ARM assembly identifier
decimal_number = pp.Combine(
pp.Optional(pp.Literal('-')) + pp.Word(pp.nums)
).setResultsName('value')
hex_number = pp.Combine(pp.Literal('0x') + pp.Word(pp.hexnums)).setResultsName('value')
relocation = pp.Combine(pp.Literal(':') + pp.Word(pp.alphanums + '_') + pp.Literal(':'))
first = pp.Word(pp.alphas + '_.', exact=1)
rest = pp.Word(pp.alphanums + '_.')
decimal_number = pp.Combine(pp.Optional(pp.Literal("-")) + pp.Word(pp.nums)).setResultsName(
"value"
)
hex_number = pp.Combine(pp.Literal("0x") + pp.Word(pp.hexnums)).setResultsName("value")
relocation = pp.Combine(pp.Literal(":") + pp.Word(pp.alphanums + "_") + pp.Literal(":"))
first = pp.Word(pp.alphas + "_.", exact=1)
rest = pp.Word(pp.alphanums + "_.")
identifier = pp.Group(
pp.Optional(relocation).setResultsName('relocation')
+ pp.Combine(first + pp.Optional(rest)).setResultsName('name')
+ pp.Optional(pp.Suppress(pp.Literal('+')) + (hex_number | decimal_number).setResultsName('offset'))
pp.Optional(relocation).setResultsName("relocation")
+ pp.Combine(first + pp.Optional(rest)).setResultsName("name")
+ pp.Optional(
pp.Suppress(pp.Literal("+"))
+ (hex_number | decimal_number).setResultsName("offset")
)
).setResultsName(self.IDENTIFIER_ID)
# Label
self.label = pp.Group(
identifier.setResultsName('name') + pp.Literal(':') + pp.Optional(self.comment)
identifier.setResultsName("name") + pp.Literal(":") + pp.Optional(self.comment)
).setResultsName(self.LABEL_ID)
# Directive
directive_option = pp.Combine(
pp.Word(pp.alphas + '#@.%', exact=1)
+ pp.Optional(pp.Word(pp.printables + ' ', excludeChars=','))
pp.Word(pp.alphas + "#@.%", exact=1)
+ pp.Optional(pp.Word(pp.printables + " ", excludeChars=","))
)
directive_parameter = (
pp.quotedString | directive_option | identifier | hex_number | decimal_number
)
commaSeparatedList = pp.delimitedList(pp.Optional(directive_parameter), delim=',')
commaSeparatedList = pp.delimitedList(pp.Optional(directive_parameter), delim=",")
self.directive = pp.Group(
pp.Literal('.')
+ pp.Word(pp.alphanums + '_').setResultsName('name')
+ (pp.OneOrMore(directive_parameter) ^ commaSeparatedList).setResultsName('parameters')
pp.Literal(".")
+ pp.Word(pp.alphanums + "_").setResultsName("name")
+ (pp.OneOrMore(directive_parameter) ^ commaSeparatedList).setResultsName("parameters")
+ pp.Optional(self.comment)
).setResultsName(self.DIRECTIVE_ID)
# LLVM-MCA markers
self.llvm_markers = pp.Group(
pp.Literal('#')
pp.Literal("#")
+ pp.Combine(
pp.CaselessLiteral('LLVM-MCA-')
+ (pp.CaselessLiteral('BEGIN') | pp.CaselessLiteral('END'))
pp.CaselessLiteral("LLVM-MCA-")
+ (pp.CaselessLiteral("BEGIN") | pp.CaselessLiteral("END"))
)
+ pp.Optional(self.comment)
).setResultsName(self.COMMENT_ID)
@@ -72,41 +75,41 @@ class ParserAArch64(BaseParser):
# Instructions
# Mnemonic
# (?P<instr>[a-zA-Z][a-zA-Z0-9]*)(?P<setflg>S?)(P?<CC>.[a-zA-Z]{2})
mnemonic = pp.Word(pp.alphanums + '.').setResultsName('mnemonic')
mnemonic = pp.Word(pp.alphanums + ".").setResultsName("mnemonic")
# Immediate:
# int: ^-?[0-9]+ | hex: ^0x[0-9a-fA-F]+ | fp: ^[0-9]{1}.[0-9]+[eE]{1}[\+-]{1}[0-9]+[fF]?
symbol_immediate = '#'
symbol_immediate = "#"
mantissa = pp.Combine(
pp.Optional(pp.Literal('-')) + pp.Word(pp.nums) + pp.Literal('.') + pp.Word(pp.nums)
).setResultsName('mantissa')
pp.Optional(pp.Literal("-")) + pp.Word(pp.nums) + pp.Literal(".") + pp.Word(pp.nums)
).setResultsName("mantissa")
exponent = (
pp.CaselessLiteral('e')
+ pp.Word('+-').setResultsName('e_sign')
+ pp.Word(pp.nums).setResultsName('exponent')
pp.CaselessLiteral("e")
+ pp.Word("+-").setResultsName("e_sign")
+ pp.Word(pp.nums).setResultsName("exponent")
)
float_ = pp.Group(
mantissa + pp.Optional(exponent) + pp.CaselessLiteral('f')
).setResultsName('float')
double_ = pp.Group(mantissa + pp.Optional(exponent)).setResultsName('double')
mantissa + pp.Optional(exponent) + pp.CaselessLiteral("f")
).setResultsName("float")
double_ = pp.Group(mantissa + pp.Optional(exponent)).setResultsName("double")
immediate = pp.Group(
pp.Optional(pp.Literal(symbol_immediate))
+ (hex_number ^ decimal_number ^ float_ ^ double_)
| (pp.Optional(pp.Literal(symbol_immediate)) + identifier)
).setResultsName(self.IMMEDIATE_ID)
shift_op = (
pp.CaselessLiteral('lsl')
^ pp.CaselessLiteral('lsr')
^ pp.CaselessLiteral('asr')
^ pp.CaselessLiteral('ror')
^ pp.CaselessLiteral('sxtw')
^ pp.CaselessLiteral('uxtw')
^ pp.CaselessLiteral('mul vl')
pp.CaselessLiteral("lsl")
^ pp.CaselessLiteral("lsr")
^ pp.CaselessLiteral("asr")
^ pp.CaselessLiteral("ror")
^ pp.CaselessLiteral("sxtw")
^ pp.CaselessLiteral("uxtw")
^ pp.CaselessLiteral("mul vl")
)
arith_immediate = pp.Group(
immediate.setResultsName('base_immediate')
+ pp.Suppress(pp.Literal(','))
+ shift_op.setResultsName('shift_op')
+ pp.Optional(immediate).setResultsName('shift')
immediate.setResultsName("base_immediate")
+ pp.Suppress(pp.Literal(","))
+ shift_op.setResultsName("shift_op")
+ pp.Optional(immediate).setResultsName("shift")
).setResultsName(self.IMMEDIATE_ID)
# Register:
# scalar: [XWBHSDQ][0-9]{1,2} | vector: [VZ][0-9]{1,2}(\.[12468]{1,2}[BHSD])?
@@ -114,83 +117,81 @@ class ParserAArch64(BaseParser):
# ignore vector len control ZCR_EL[123] for now
# define SP, ZR register aliases as regex, due to pyparsing does not support
# proper lookahead
alias_r31_sp = pp.Regex('(?P<prefix>[a-zA-Z])?(?P<name>(sp|SP))')
alias_r31_zr = pp.Regex('(?P<prefix>[a-zA-Z])?(?P<name>(zr|ZR))')
scalar = pp.Word('xwbhsdqXWBHSDQ', exact=1).setResultsName('prefix') + pp.Word(
alias_r31_sp = pp.Regex("(?P<prefix>[a-zA-Z])?(?P<name>(sp|SP))")
alias_r31_zr = pp.Regex("(?P<prefix>[a-zA-Z])?(?P<name>(zr|ZR))")
scalar = pp.Word("xwbhsdqXWBHSDQ", exact=1).setResultsName("prefix") + pp.Word(
pp.nums
).setResultsName('name')
index = pp.Literal('[') + pp.Word(pp.nums).setResultsName('index') + pp.Literal(']')
).setResultsName("name")
index = pp.Literal("[") + pp.Word(pp.nums).setResultsName("index") + pp.Literal("]")
vector = (
pp.oneOf('v z', caseless=True).setResultsName('prefix')
+ pp.Word(pp.nums).setResultsName('name')
pp.oneOf("v z", caseless=True).setResultsName("prefix")
+ pp.Word(pp.nums).setResultsName("name")
+ pp.Optional(
pp.Literal('.')
+ pp.Optional(pp.Word('12468')).setResultsName('lanes')
+ pp.Word(pp.alphas, exact=1).setResultsName('shape')
pp.Literal(".")
+ pp.Optional(pp.Word("12468")).setResultsName("lanes")
+ pp.Word(pp.alphas, exact=1).setResultsName("shape")
+ pp.Optional(index)
)
)
predicate = (
pp.CaselessLiteral('p').setResultsName('prefix')
+ pp.Word(pp.nums).setResultsName('name')
pp.CaselessLiteral("p").setResultsName("prefix")
+ pp.Word(pp.nums).setResultsName("name")
+ pp.Optional(
(
pp.Suppress(pp.Literal('/'))
+ pp.oneOf('z m', caseless=True).setResultsName('predication')
pp.Suppress(pp.Literal("/"))
+ pp.oneOf("z m", caseless=True).setResultsName("predication")
)
| (
pp.Literal('.')
+ pp.Optional(pp.Word('12468')).setResultsName('lanes')
+ pp.Word(pp.alphas, exact=1).setResultsName('shape')
pp.Literal(".")
+ pp.Optional(pp.Word("12468")).setResultsName("lanes")
+ pp.Word(pp.alphas, exact=1).setResultsName("shape")
)
)
)
self.list_element = vector ^ scalar
register_list = (
pp.Literal('{')
pp.Literal("{")
+ (
pp.delimitedList(pp.Combine(self.list_element), delim=',').setResultsName('list')
^ pp.delimitedList(pp.Combine(self.list_element), delim='-').setResultsName(
'range'
pp.delimitedList(pp.Combine(self.list_element), delim=",").setResultsName("list")
^ pp.delimitedList(pp.Combine(self.list_element), delim="-").setResultsName("range")
)
)
+ pp.Literal('}')
+ pp.Literal("}")
+ pp.Optional(index)
)
register = pp.Group(
(alias_r31_sp | alias_r31_zr | vector | scalar | predicate | register_list)
# (alias_r31_sp | alias_r31_zr | vector | scalar | predicate | register_list)
+ pp.Optional(
pp.Suppress(pp.Literal(','))
+ shift_op.setResultsName('shift_op')
+ pp.Optional(immediate).setResultsName('shift')
pp.Suppress(pp.Literal(","))
+ shift_op.setResultsName("shift_op")
+ pp.Optional(immediate).setResultsName("shift")
)
).setResultsName(self.REGISTER_ID)
self.register = register
# Memory
register_index = register.setResultsName('index') + pp.Optional(
pp.Literal(',') + pp.Word(pp.alphas) + immediate.setResultsName('scale')
register_index = register.setResultsName("index") + pp.Optional(
pp.Literal(",") + pp.Word(pp.alphas) + immediate.setResultsName("scale")
)
memory = pp.Group(
pp.Literal('[')
+ pp.Optional(register.setResultsName('base'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(register_index ^ (immediate ^ arith_immediate).setResultsName('offset'))
+ pp.Literal(']')
pp.Literal("[")
+ pp.Optional(register.setResultsName("base"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(register_index ^ (immediate ^ arith_immediate).setResultsName("offset"))
+ pp.Literal("]")
+ pp.Optional(
pp.Literal('!').setResultsName('pre_indexed')
| (pp.Suppress(pp.Literal(',')) + immediate.setResultsName('post_indexed'))
pp.Literal("!").setResultsName("pre_indexed")
| (pp.Suppress(pp.Literal(",")) + immediate.setResultsName("post_indexed"))
)
).setResultsName(self.MEMORY_ID)
prefetch_op = pp.Group(
pp.Group(pp.CaselessLiteral('PLD') ^ pp.CaselessLiteral('PST')).setResultsName('type')
pp.Group(pp.CaselessLiteral("PLD") ^ pp.CaselessLiteral("PST")).setResultsName("type")
+ pp.Group(
pp.CaselessLiteral('L1') ^ pp.CaselessLiteral('L2') ^ pp.CaselessLiteral('L3')
).setResultsName('target')
+ pp.Group(pp.CaselessLiteral('KEEP') ^ pp.CaselessLiteral('STRM')).setResultsName(
'policy'
pp.CaselessLiteral("L1") ^ pp.CaselessLiteral("L2") ^ pp.CaselessLiteral("L3")
).setResultsName("target")
+ pp.Group(pp.CaselessLiteral("KEEP") ^ pp.CaselessLiteral("STRM")).setResultsName(
"policy"
)
).setResultsName('prfop')
).setResultsName("prfop")
# Combine to instruction form
operand_first = pp.Group(
register ^ (prefetch_op | immediate) ^ memory ^ arith_immediate ^ identifier
@@ -198,15 +199,15 @@ class ParserAArch64(BaseParser):
operand_rest = pp.Group((register ^ immediate ^ memory ^ arith_immediate) | identifier)
self.instruction_parser = (
mnemonic
+ pp.Optional(operand_first.setResultsName('operand1'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand2'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand3'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand4'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand5'))
+ pp.Optional(operand_first.setResultsName("operand1"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand2"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand3"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand4"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand5"))
+ pp.Optional(self.comment)
)
@@ -231,8 +232,8 @@ class ParserAArch64(BaseParser):
self.DIRECTIVE_ID: None,
self.COMMENT_ID: None,
self.LABEL_ID: None,
'line': line,
'line_number': line_number,
"line": line,
"line_number": line_number,
}
)
result = None
@@ -241,7 +242,7 @@ class ParserAArch64(BaseParser):
try:
result = self.process_operand(self.comment.parseString(line, parseAll=True).asDict())
result = AttrDict.convert_dict(result)
instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID])
instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID])
except pp.ParseException:
pass
# 1.2 check for llvm-mca marker
@@ -250,7 +251,7 @@ class ParserAArch64(BaseParser):
self.llvm_markers.parseString(line, parseAll=True).asDict()
)
result = AttrDict.convert_dict(result)
instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID])
instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID])
except pp.ParseException:
pass
# 2. Parse label
@@ -260,7 +261,7 @@ class ParserAArch64(BaseParser):
result = AttrDict.convert_dict(result)
instruction_form[self.LABEL_ID] = result[self.LABEL_ID].name
if self.COMMENT_ID in result[self.LABEL_ID]:
instruction_form[self.COMMENT_ID] = ' '.join(
instruction_form[self.COMMENT_ID] = " ".join(
result[self.LABEL_ID][self.COMMENT_ID]
)
except pp.ParseException:
@@ -275,12 +276,12 @@ class ParserAArch64(BaseParser):
result = AttrDict.convert_dict(result)
instruction_form[self.DIRECTIVE_ID] = AttrDict(
{
'name': result[self.DIRECTIVE_ID].name,
'parameters': result[self.DIRECTIVE_ID].parameters,
"name": result[self.DIRECTIVE_ID].name,
"parameters": result[self.DIRECTIVE_ID].parameters,
}
)
if self.COMMENT_ID in result[self.DIRECTIVE_ID]:
instruction_form[self.COMMENT_ID] = ' '.join(
instruction_form[self.COMMENT_ID] = " ".join(
result[self.DIRECTIVE_ID][self.COMMENT_ID]
)
except pp.ParseException:
@@ -292,8 +293,7 @@ class ParserAArch64(BaseParser):
result = self.parse_instruction(line)
except (pp.ParseException, KeyError) as e:
raise e
raise ValueError(
'Unable to parse {!r} on line {}'.format(line, line_number)) from e
raise ValueError("Unable to parse {!r} on line {}".format(line, line_number)) from e
instruction_form[self.INSTRUCTION_ID] = result[self.INSTRUCTION_ID]
instruction_form[self.OPERANDS_ID] = result[self.OPERANDS_ID]
instruction_form[self.COMMENT_ID] = result[self.COMMENT_ID]
@@ -312,26 +312,26 @@ class ParserAArch64(BaseParser):
operands = []
# Add operands to list
# Check first operand
if 'operand1' in result:
operands.append(self.process_operand(result['operand1']))
if "operand1" in result:
operands.append(self.process_operand(result["operand1"]))
# Check second operand
if 'operand2' in result:
operands.append(self.process_operand(result['operand2']))
if "operand2" in result:
operands.append(self.process_operand(result["operand2"]))
# Check third operand
if 'operand3' in result:
operands.append(self.process_operand(result['operand3']))
if "operand3" in result:
operands.append(self.process_operand(result["operand3"]))
# Check fourth operand
if 'operand4' in result:
operands.append(self.process_operand(result['operand4']))
if "operand4" in result:
operands.append(self.process_operand(result["operand4"]))
# Check fifth operand
if 'operand5' in result:
operands.append(self.process_operand(result['operand5']))
if "operand5" in result:
operands.append(self.process_operand(result["operand5"]))
return_dict = AttrDict(
{
self.INSTRUCTION_ID: result.mnemonic,
self.OPERANDS_ID: operands,
self.COMMENT_ID: ' '.join(result[self.COMMENT_ID])
self.COMMENT_ID: " ".join(result[self.COMMENT_ID])
if self.COMMENT_ID in result
else None,
}
@@ -345,11 +345,11 @@ class ParserAArch64(BaseParser):
return self.process_memory_address(operand[self.MEMORY_ID])
# structure register lists
if self.REGISTER_ID in operand and (
'list' in operand[self.REGISTER_ID] or 'range' in operand[self.REGISTER_ID]
"list" in operand[self.REGISTER_ID] or "range" in operand[self.REGISTER_ID]
):
# TODO: discuss if ranges should be converted to lists
return self.process_register_list(operand[self.REGISTER_ID])
if self.REGISTER_ID in operand and operand[self.REGISTER_ID]['name'] == 'sp':
if self.REGISTER_ID in operand and operand[self.REGISTER_ID]["name"] == "sp":
return self.process_sp_register(operand[self.REGISTER_ID])
# add value attribute to floating point immediates without exponent
if self.IMMEDIATE_ID in operand:
@@ -363,140 +363,140 @@ class ParserAArch64(BaseParser):
def process_memory_address(self, memory_address):
"""Post-process memory address operand"""
# Remove unnecessarily created dictionary entries during parsing
offset = memory_address.get('offset', None)
offset = memory_address.get("offset", None)
if isinstance(offset, list) and len(offset) == 1:
offset = offset[0]
base = memory_address.get('base', None)
index = memory_address.get('index', None)
base = memory_address.get("base", None)
index = memory_address.get("index", None)
scale = 1
if base is not None and 'name' in base and base['name'] == 'sp':
base['prefix'] = 'x'
if index is not None and 'name' in index and index['name'] == 'sp':
index['prefix'] = 'x'
valid_shift_ops = ['lsl', 'uxtw', 'sxtw']
if 'index' in memory_address:
if 'shift' in memory_address['index']:
if memory_address['index']['shift_op'].lower() in valid_shift_ops:
scale = 2 ** int(memory_address['index']['shift'][0]['value'])
new_dict = AttrDict({'offset': offset, 'base': base, 'index': index, 'scale': scale})
if 'pre_indexed' in memory_address:
new_dict['pre_indexed'] = True
if 'post_indexed' in memory_address:
new_dict['post_indexed'] = memory_address['post_indexed']
if base is not None and "name" in base and base["name"] == "sp":
base["prefix"] = "x"
if index is not None and "name" in index and index["name"] == "sp":
index["prefix"] = "x"
valid_shift_ops = ["lsl", "uxtw", "sxtw"]
if "index" in memory_address:
if "shift" in memory_address["index"]:
if memory_address["index"]["shift_op"].lower() in valid_shift_ops:
scale = 2 ** int(memory_address["index"]["shift"][0]["value"])
new_dict = AttrDict({"offset": offset, "base": base, "index": index, "scale": scale})
if "pre_indexed" in memory_address:
new_dict["pre_indexed"] = True
if "post_indexed" in memory_address:
new_dict["post_indexed"] = memory_address["post_indexed"]
return AttrDict({self.MEMORY_ID: new_dict})
def process_sp_register(self, register):
"""Post-process stack pointer register"""
reg = register
reg['prefix'] = 'x'
reg["prefix"] = "x"
return AttrDict({self.REGISTER_ID: reg})
def process_register_list(self, register_list):
"""Post-process register lists (e.g., {r0,r3,r5}) and register ranges (e.g., {r0-r7})"""
# Remove unnecessarily created dictionary entries during parsing
rlist = []
dict_name = ''
if 'list' in register_list:
dict_name = 'list'
if 'range' in register_list:
dict_name = 'range'
dict_name = ""
if "list" in register_list:
dict_name = "list"
if "range" in register_list:
dict_name = "range"
for r in register_list[dict_name]:
rlist.append(
AttrDict.convert_dict(self.list_element.parseString(r, parseAll=True).asDict())
)
index = register_list.get('index', None)
new_dict = AttrDict({dict_name: rlist, 'index': index})
index = register_list.get("index", None)
new_dict = AttrDict({dict_name: rlist, "index": index})
if len(new_dict[dict_name]) == 1:
return AttrDict({self.REGISTER_ID: new_dict[dict_name][0]})
return AttrDict({self.REGISTER_ID: new_dict})
def process_immediate(self, immediate):
"""Post-process immediate operand"""
dict_name = ''
if 'identifier' in immediate:
dict_name = ""
if "identifier" in immediate:
# actually an identifier, change declaration
return immediate
if 'value' in immediate:
if "value" in immediate:
# normal integer value, nothing to do
return AttrDict({self.IMMEDIATE_ID: immediate})
if 'base_immediate' in immediate:
if "base_immediate" in immediate:
# arithmetic immediate, add calculated value as value
immediate['shift'] = immediate['shift'][0]
immediate['value'] = (
int(immediate['base_immediate']['value'], 0) << int(immediate['shift']['value'])
immediate["shift"] = immediate["shift"][0]
immediate["value"] = int(immediate["base_immediate"]["value"], 0) << int(
immediate["shift"]["value"]
)
return AttrDict({self.IMMEDIATE_ID: immediate})
if 'float' in immediate:
dict_name = 'float'
if 'double' in immediate:
dict_name = 'double'
if 'exponent' in immediate[dict_name]:
if "float" in immediate:
dict_name = "float"
if "double" in immediate:
dict_name = "double"
if "exponent" in immediate[dict_name]:
# nothing to do
return AttrDict({self.IMMEDIATE_ID: immediate})
else:
# change 'mantissa' key to 'value'
return AttrDict(
{self.IMMEDIATE_ID: AttrDict({'value': immediate[dict_name]['mantissa']})}
{self.IMMEDIATE_ID: AttrDict({"value": immediate[dict_name]["mantissa"]})}
)
def process_label(self, label):
"""Post-process label asm line"""
# remove duplicated 'name' level due to identifier
label['name'] = label['name']['name']
label["name"] = label["name"]["name"]
return AttrDict({self.LABEL_ID: label})
def process_identifier(self, identifier):
"""Post-process identifier operand"""
# remove value if it consists of symbol+offset
if 'value' in identifier:
del identifier['value']
if "value" in identifier:
del identifier["value"]
return AttrDict({self.IDENTIFIER_ID: identifier})
def get_full_reg_name(self, register):
"""Return one register name string including all attributes"""
if 'lanes' in register:
if "lanes" in register:
return (
register['prefix']
+ str(register['name'])
+ '.'
+ str(register['lanes'])
+ register['shape']
register["prefix"]
+ str(register["name"])
+ "."
+ str(register["lanes"])
+ register["shape"]
)
return register['prefix'] + str(register['name'])
return register["prefix"] + str(register["name"])
def normalize_imd(self, imd):
"""Normalize immediate to decimal based representation"""
if 'value' in imd:
if imd['value'].lower().startswith('0x'):
if "value" in imd:
if imd["value"].lower().startswith("0x"):
# hex, return decimal
return int(imd['value'], 16)
return int(imd['value'], 10)
elif 'float' in imd:
return self.ieee_to_float(imd['float'])
elif 'double' in imd:
return self.ieee_to_float(imd['double'])
return int(imd["value"], 16)
return int(imd["value"], 10)
elif "float" in imd:
return self.ieee_to_float(imd["float"])
elif "double" in imd:
return self.ieee_to_float(imd["double"])
# identifier
return imd
def ieee_to_float(self, ieee_val):
"""Convert IEEE representation to python float"""
exponent = int(ieee_val['exponent'], 10)
if ieee_val['e_sign'] == '-':
exponent = int(ieee_val["exponent"], 10)
if ieee_val["e_sign"] == "-":
exponent *= -1
return float(ieee_val['mantissa']) * (10 ** exponent)
return float(ieee_val["mantissa"]) * (10 ** exponent)
def parse_register(self, register_string):
raise NotImplementedError
def is_gpr(self, register):
"""Check if register is a general purpose register"""
if register['prefix'] in 'wx':
if register["prefix"] in "wx":
return True
return False
def is_vector_register(self, register):
"""Check if register is a vector register"""
if register['prefix'] in 'bhsdqvz':
if register["prefix"] in "bhsdqvz":
return True
return False
@@ -510,15 +510,15 @@ class ParserAArch64(BaseParser):
def is_reg_dependend_of(self, reg_a, reg_b):
"""Check if ``reg_a`` is dependent on ``reg_b``"""
prefixes_gpr = 'wx'
prefixes_vec = 'bhsdqvz'
if reg_a['name'] == reg_b['name']:
if reg_a['prefix'].lower() in prefixes_gpr and reg_b['prefix'].lower() in prefixes_gpr:
prefixes_gpr = "wx"
prefixes_vec = "bhsdqvz"
if reg_a["name"] == reg_b["name"]:
if reg_a["prefix"].lower() in prefixes_gpr and reg_b["prefix"].lower() in prefixes_gpr:
return True
if reg_a['prefix'].lower() in prefixes_vec and reg_b['prefix'].lower() in prefixes_vec:
if reg_a["prefix"].lower() in prefixes_vec and reg_b["prefix"].lower() in prefixes_vec:
return True
return False
def get_reg_type(self, register):
"""Get register type"""
return register['prefix']
return register["prefix"]

View File

@@ -19,67 +19,70 @@ class ParserX86ATT(BaseParser):
def __init__(self):
super().__init__()
self.isa = 'x86'
self.isa = "x86"
def construct_parser(self):
"""Create parser for ARM AArch64 ISA."""
decimal_number = pp.Combine(
pp.Optional(pp.Literal('-')) + pp.Word(pp.nums)
).setResultsName('value')
hex_number = pp.Combine(pp.Optional(pp.Literal('-')) + pp.Literal('0x') + pp.Word(pp.hexnums)).setResultsName('value')
decimal_number = pp.Combine(pp.Optional(pp.Literal("-")) + pp.Word(pp.nums)).setResultsName(
"value"
)
hex_number = pp.Combine(
pp.Optional(pp.Literal("-")) + pp.Literal("0x") + pp.Word(pp.hexnums)
).setResultsName("value")
# Comment - either '#' or '//' (icc)
self.comment = (pp.Literal('#') | pp.Literal('//')) + pp.Group(
self.comment = (pp.Literal("#") | pp.Literal("//")) + pp.Group(
pp.ZeroOrMore(pp.Word(pp.printables))
).setResultsName(self.COMMENT_ID)
# Define x86 assembly identifier
relocation = pp.Combine(pp.Literal('@') + pp.Word(pp.alphas))
id_offset = pp.Word(pp.nums) + pp.Suppress(pp.Literal('+'))
first = pp.Word(pp.alphas + '_.', exact=1)
rest = pp.Word(pp.alphanums + '$_.+-')
relocation = pp.Combine(pp.Literal("@") + pp.Word(pp.alphas))
id_offset = pp.Word(pp.nums) + pp.Suppress(pp.Literal("+"))
first = pp.Word(pp.alphas + "_.", exact=1)
rest = pp.Word(pp.alphanums + "$_.+-")
identifier = pp.Group(
pp.Optional(id_offset).setResultsName('offset')
+ pp.Combine(pp.delimitedList(
pp.Combine(first + pp.Optional(rest)), delim='::'), joinString='::'
).setResultsName('name')
+ pp.Optional(relocation).setResultsName('relocation')
).setResultsName('identifier')
pp.Optional(id_offset).setResultsName("offset")
+ pp.Combine(
pp.delimitedList(pp.Combine(first + pp.Optional(rest)), delim="::"), joinString="::"
).setResultsName("name")
+ pp.Optional(relocation).setResultsName("relocation")
).setResultsName("identifier")
# Label
label_rest = pp.Word(pp.alphanums + '$_.+-()')
label_rest = pp.Word(pp.alphanums + "$_.+-()")
label_identifier = pp.Group(
pp.Optional(id_offset).setResultsName('offset')
+ pp.Combine(pp.delimitedList(
pp.Combine(first + pp.Optional(label_rest)), delim='::'), joinString='::'
).setResultsName('name')
+ pp.Optional(relocation).setResultsName('relocation')
).setResultsName('identifier')
pp.Optional(id_offset).setResultsName("offset")
+ pp.Combine(
pp.delimitedList(pp.Combine(first + pp.Optional(label_rest)), delim="::"),
joinString="::",
).setResultsName("name")
+ pp.Optional(relocation).setResultsName("relocation")
).setResultsName("identifier")
numeric_identifier = pp.Group(
pp.Word(pp.nums).setResultsName('name')
+ pp.Optional(pp.oneOf('b f', caseless=True).setResultsName('suffix'))
).setResultsName('identifier')
pp.Word(pp.nums).setResultsName("name")
+ pp.Optional(pp.oneOf("b f", caseless=True).setResultsName("suffix"))
).setResultsName("identifier")
self.label = pp.Group(
(label_identifier | numeric_identifier).setResultsName('name')
+ pp.Literal(':')
(label_identifier | numeric_identifier).setResultsName("name")
+ pp.Literal(":")
+ pp.Optional(self.comment)
).setResultsName(self.LABEL_ID)
# Register: pp.Regex('^%[0-9a-zA-Z]+{}{z},?')
self.register = pp.Group(
pp.Literal('%')
+ pp.Word(pp.alphanums).setResultsName('name')
+ pp.Optional(pp.Literal('(') + pp.Word(pp.nums) + pp.Literal(')'))
pp.Literal("%")
+ pp.Word(pp.alphanums).setResultsName("name")
+ pp.Optional(pp.Literal("(") + pp.Word(pp.nums) + pp.Literal(")"))
+ pp.Optional(
pp.Literal('{')
+ pp.Optional(pp.Suppress(pp.Literal('%')))
+ pp.Word(pp.alphanums).setResultsName('mask')
+ pp.Literal('}')
pp.Literal("{")
+ pp.Optional(pp.Suppress(pp.Literal("%")))
+ pp.Word(pp.alphanums).setResultsName("mask")
+ pp.Literal("}")
+ pp.Optional(
pp.Suppress(pp.Literal('{'))
+ pp.Literal('z').setResultsName('zeroing')
+ pp.Suppress(pp.Literal('}'))
pp.Suppress(pp.Literal("{"))
+ pp.Literal("z").setResultsName("zeroing")
+ pp.Suppress(pp.Literal("}"))
)
)
).setResultsName(self.REGISTER_ID)
# Immediate: pp.Regex('^\$(-?[0-9]+)|(0x[0-9a-fA-F]+),?')
symbol_immediate = '$'
symbol_immediate = "$"
immediate = pp.Group(
pp.Literal(symbol_immediate) + (hex_number | decimal_number | identifier)
).setResultsName(self.IMMEDIATE_ID)
@@ -88,53 +91,52 @@ class ParserX86ATT(BaseParser):
offset = pp.Group(identifier | hex_number | decimal_number).setResultsName(
self.IMMEDIATE_ID
)
scale = pp.Word('1248', exact=1)
scale = pp.Word("1248", exact=1)
# Segment register extension
segment_extension = (
hex_number
^ pp.Word(pp.nums)
^ pp.Group(
pp.Optional(offset.setResultsName('offset'))
+ pp.Literal('(')
+ pp.Optional(self.register.setResultsName('base'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(self.register.setResultsName('index'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(scale.setResultsName('scale'))
+ pp.Literal(')')
pp.Optional(offset.setResultsName("offset"))
+ pp.Literal("(")
+ pp.Optional(self.register.setResultsName("base"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(self.register.setResultsName("index"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(scale.setResultsName("scale"))
+ pp.Literal(")")
)
)
memory_segmentation = (
self.register.setResultsName('base')
+ pp.Literal(':')
self.register.setResultsName("base")
+ pp.Literal(":")
+ segment_extension.setResultsName(self.SEGMENT_EXT_ID)
)
# Memory: offset | seg:seg_ext | offset(base, index, scale){mask}
memory_abs = (
pp.Suppress(pp.Literal('*'))
+ (offset | self.register).setResultsName('offset')
memory_abs = pp.Suppress(pp.Literal("*")) + (offset | self.register).setResultsName(
"offset"
)
memory = pp.Group(
(
pp.Optional(pp.Suppress(pp.Literal('*')))
+ pp.Optional(offset.setResultsName('offset'))
+ pp.Literal('(')
+ pp.Optional(self.register.setResultsName('base'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(self.register.setResultsName('index'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(scale.setResultsName('scale'))
+ pp.Literal(')')
pp.Optional(pp.Suppress(pp.Literal("*")))
+ pp.Optional(offset.setResultsName("offset"))
+ pp.Literal("(")
+ pp.Optional(self.register.setResultsName("base"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(self.register.setResultsName("index"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(scale.setResultsName("scale"))
+ pp.Literal(")")
+ pp.Optional(
pp.Literal('{')
+ pp.Optional(pp.Suppress(pp.Literal('%')))
+ pp.Word(pp.alphanums).setResultsName('mask')
+ pp.Literal('}')
pp.Literal("{")
+ pp.Optional(pp.Suppress(pp.Literal("%")))
+ pp.Word(pp.alphanums).setResultsName("mask")
+ pp.Literal("}")
)
)
| memory_abs
| memory_segmentation
| (hex_number | pp.Word(pp.nums)).setResultsName('offset')
| (hex_number | pp.Word(pp.nums)).setResultsName("offset")
).setResultsName(self.MEMORY_ID)
# Directive
@@ -143,23 +145,23 @@ class ParserX86ATT(BaseParser):
directive_parameter = (
pp.quotedString
^ (
pp.Word(pp.printables, excludeChars=',#')
+ pp.Optional(pp.Suppress(pp.Literal(',')))
pp.Word(pp.printables, excludeChars=",#")
+ pp.Optional(pp.Suppress(pp.Literal(",")))
)
^ pp.Suppress(pp.Literal(','))
^ pp.Suppress(pp.Literal(","))
)
self.directive = pp.Group(
pp.Literal('.')
+ pp.Word(pp.alphanums + '_').setResultsName('name')
+ pp.ZeroOrMore(directive_parameter).setResultsName('parameters')
pp.Literal(".")
+ pp.Word(pp.alphanums + "_").setResultsName("name")
+ pp.ZeroOrMore(directive_parameter).setResultsName("parameters")
+ pp.Optional(self.comment)
).setResultsName(self.DIRECTIVE_ID)
# Instructions
# Mnemonic
mnemonic = pp.ZeroOrMore(pp.Literal('data16') | pp.Literal('data32')) + pp.Word(
pp.alphanums + ','
).setResultsName('mnemonic')
mnemonic = pp.ZeroOrMore(pp.Literal("data16") | pp.Literal("data32")) + pp.Word(
pp.alphanums + ","
).setResultsName("mnemonic")
# Combine to instruction form
operand_first = pp.Group(
self.register ^ immediate ^ memory ^ identifier ^ numeric_identifier
@@ -167,13 +169,13 @@ class ParserX86ATT(BaseParser):
operand_rest = pp.Group(self.register ^ immediate ^ memory)
self.instruction_parser = (
mnemonic
+ pp.Optional(operand_first.setResultsName('operand1'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand2'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand3'))
+ pp.Optional(pp.Suppress(pp.Literal(',')))
+ pp.Optional(operand_rest.setResultsName('operand4'))
+ pp.Optional(operand_first.setResultsName("operand1"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand2"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand3"))
+ pp.Optional(pp.Suppress(pp.Literal(",")))
+ pp.Optional(operand_rest.setResultsName("operand4"))
+ pp.Optional(self.comment)
)
@@ -202,8 +204,8 @@ class ParserX86ATT(BaseParser):
self.DIRECTIVE_ID: None,
self.COMMENT_ID: None,
self.LABEL_ID: None,
'line': line,
'line_number': line_number,
"line": line,
"line_number": line_number,
}
)
result = None
@@ -212,7 +214,7 @@ class ParserX86ATT(BaseParser):
try:
result = self.process_operand(self.comment.parseString(line, parseAll=True).asDict())
result = AttrDict.convert_dict(result)
instruction_form[self.COMMENT_ID] = ' '.join(result[self.COMMENT_ID])
instruction_form[self.COMMENT_ID] = " ".join(result[self.COMMENT_ID])
except pp.ParseException:
pass
@@ -221,9 +223,9 @@ class ParserX86ATT(BaseParser):
try:
result = self.process_operand(self.label.parseString(line, parseAll=True).asDict())
result = AttrDict.convert_dict(result)
instruction_form[self.LABEL_ID] = result[self.LABEL_ID]['name']
instruction_form[self.LABEL_ID] = result[self.LABEL_ID]["name"]
if self.COMMENT_ID in result[self.LABEL_ID]:
instruction_form[self.COMMENT_ID] = ' '.join(
instruction_form[self.COMMENT_ID] = " ".join(
result[self.LABEL_ID][self.COMMENT_ID]
)
except pp.ParseException:
@@ -238,12 +240,12 @@ class ParserX86ATT(BaseParser):
result = AttrDict.convert_dict(result)
instruction_form[self.DIRECTIVE_ID] = AttrDict(
{
'name': result[self.DIRECTIVE_ID]['name'],
'parameters': result[self.DIRECTIVE_ID]['parameters'],
"name": result[self.DIRECTIVE_ID]["name"],
"parameters": result[self.DIRECTIVE_ID]["parameters"],
}
)
if self.COMMENT_ID in result[self.DIRECTIVE_ID]:
instruction_form[self.COMMENT_ID] = ' '.join(
instruction_form[self.COMMENT_ID] = " ".join(
result[self.DIRECTIVE_ID][self.COMMENT_ID]
)
except pp.ParseException:
@@ -255,7 +257,7 @@ class ParserX86ATT(BaseParser):
result = self.parse_instruction(line)
except pp.ParseException:
raise ValueError(
'Could not parse instruction on line {}: {!r}'.format(line_number, line)
"Could not parse instruction on line {}: {!r}".format(line_number, line)
)
instruction_form[self.INSTRUCTION_ID] = result[self.INSTRUCTION_ID]
instruction_form[self.OPERANDS_ID] = result[self.OPERANDS_ID]
@@ -275,22 +277,22 @@ class ParserX86ATT(BaseParser):
operands = []
# Add operands to list
# Check first operand
if 'operand1' in result:
operands.append(self.process_operand(result['operand1']))
if "operand1" in result:
operands.append(self.process_operand(result["operand1"]))
# Check second operand
if 'operand2' in result:
operands.append(self.process_operand(result['operand2']))
if "operand2" in result:
operands.append(self.process_operand(result["operand2"]))
# Check third operand
if 'operand3' in result:
operands.append(self.process_operand(result['operand3']))
if "operand3" in result:
operands.append(self.process_operand(result["operand3"]))
# Check fourth operand
if 'operand4' in result:
operands.append(self.process_operand(result['operand4']))
if "operand4" in result:
operands.append(self.process_operand(result["operand4"]))
return_dict = AttrDict(
{
self.INSTRUCTION_ID: result['mnemonic'].split(',')[0],
self.INSTRUCTION_ID: result["mnemonic"].split(",")[0],
self.OPERANDS_ID: operands,
self.COMMENT_ID: ' '.join(result[self.COMMENT_ID])
self.COMMENT_ID: " ".join(result[self.COMMENT_ID])
if self.COMMENT_ID in result
else None,
}
@@ -311,23 +313,23 @@ class ParserX86ATT(BaseParser):
return operand
def process_directive(self, directive):
directive_new = {'name': directive['name'], 'parameters': []}
if 'parameters' in directive:
directive_new['parameters'] = directive['parameters']
if 'comment' in directive:
directive_new['comment'] = directive['comment']
directive_new = {"name": directive["name"], "parameters": []}
if "parameters" in directive:
directive_new["parameters"] = directive["parameters"]
if "comment" in directive:
directive_new["comment"] = directive["comment"]
return AttrDict({self.DIRECTIVE_ID: directive_new})
def process_memory_address(self, memory_address):
"""Post-process memory address operand"""
# Remove unecessarily created dictionary entries during memory address parsing
offset = memory_address.get('offset', None)
base = memory_address.get('base', None)
index = memory_address.get('index', None)
scale = 1 if 'scale' not in memory_address else int(memory_address['scale'])
offset = memory_address.get("offset", None)
base = memory_address.get("base", None)
index = memory_address.get("index", None)
scale = 1 if "scale" not in memory_address else int(memory_address["scale"])
if isinstance(offset, str) and base is None and index is None:
offset = {'value': offset}
new_dict = AttrDict({'offset': offset, 'base': base, 'index': index, 'scale': scale})
offset = {"value": offset}
new_dict = AttrDict({"offset": offset, "base": base, "index": index, "scale": scale})
# Add segmentation extension if existing
if self.SEGMENT_EXT_ID in memory_address:
new_dict[self.SEGMENT_EXT_ID] = memory_address[self.SEGMENT_EXT_ID]
@@ -336,12 +338,12 @@ class ParserX86ATT(BaseParser):
def process_label(self, label):
"""Post-process label asm line"""
# remove duplicated 'name' level due to identifier
label['name'] = label['name'][0]['name']
label["name"] = label["name"][0]["name"]
return AttrDict({self.LABEL_ID: label})
def process_immediate(self, immediate):
"""Post-process immediate operand"""
if 'identifier' in immediate:
if "identifier" in immediate:
# actually an identifier, change declaration
return immediate
# otherwise nothing to do
@@ -350,15 +352,15 @@ class ParserX86ATT(BaseParser):
def get_full_reg_name(self, register):
"""Return one register name string including all attributes"""
# nothing to do
return register['name']
return register["name"]
def normalize_imd(self, imd):
"""Normalize immediate to decimal based representation"""
if 'value' in imd:
if imd['value'].lower().startswith('0x'):
if "value" in imd:
if imd["value"].lower().startswith("0x"):
# hex, return decimal
return int(imd['value'], 16)
return int(imd['value'], 10)
return int(imd["value"], 16)
return int(imd["value"], 10)
# identifier
return imd
@@ -373,8 +375,8 @@ class ParserX86ATT(BaseParser):
def is_reg_dependend_of(self, reg_a, reg_b):
"""Check if ``reg_a`` is dependent on ``reg_b``"""
# Normalize name
reg_a_name = reg_a['name'].upper()
reg_b_name = reg_b['name'].upper()
reg_a_name = reg_a["name"].upper()
reg_b_name = reg_b["name"].upper()
# Check if they are the same registers
if reg_a_name == reg_b_name:
@@ -388,13 +390,13 @@ class ParserX86ATT(BaseParser):
return False
# Check basic GPRs
gpr_groups = {
'A': ['RAX', 'EAX', 'AX', 'AH', 'AL'],
'B': ['RBX', 'EBX', 'BX', 'BH', 'BL'],
'C': ['RCX', 'ECX', 'CX', 'CH', 'CL'],
'D': ['RDX', 'EDX', 'DX', 'DH', 'DL'],
'SP': ['RSP', 'ESP', 'SP', 'SPL'],
'SRC': ['RSI', 'ESI', 'SI', 'SIL'],
'DST': ['RDI', 'EDI', 'DI', 'DIL']
"A": ["RAX", "EAX", "AX", "AH", "AL"],
"B": ["RBX", "EBX", "BX", "BH", "BL"],
"C": ["RCX", "ECX", "CX", "CH", "CL"],
"D": ["RDX", "EDX", "DX", "DH", "DL"],
"SP": ["RSP", "ESP", "SP", "SPL"],
"SRC": ["RSI", "ESI", "SI", "SIL"],
"DST": ["RDI", "EDI", "DI", "DIL"],
}
if self.is_basic_gpr(reg_a):
if self.is_basic_gpr(reg_b):
@@ -405,8 +407,8 @@ class ParserX86ATT(BaseParser):
return False
# Check other GPRs
ma = re.match(r'R([0-9]+)[DWB]?', reg_a_name)
mb = re.match(r'R([0-9]+)[DWB]?', reg_b_name)
ma = re.match(r"R([0-9]+)[DWB]?", reg_a_name)
mb = re.match(r"R([0-9]+)[DWB]?", reg_b_name)
if ma and mb and ma.group(1) == mb.group(1):
return True
@@ -415,9 +417,8 @@ class ParserX86ATT(BaseParser):
def is_basic_gpr(self, register):
"""Check if register is a basic general purpose register (ebi, rax, ...)"""
if (
any(char.isdigit() for char in register['name'])
or any(register['name'].lower().startswith(x) for x in ['mm', 'xmm', 'ymm', 'zmm'])
if any(char.isdigit() for char in register["name"]) or any(
register["name"].lower().startswith(x) for x in ["mm", "xmm", "ymm", "zmm"]
):
return False
return True
@@ -428,13 +429,13 @@ class ParserX86ATT(BaseParser):
return False
if self.is_basic_gpr(register):
return True
return re.match(r'R([0-9]+)[DWB]?', register['name'], re.IGNORECASE)
return re.match(r"R([0-9]+)[DWB]?", register["name"], re.IGNORECASE)
def is_vector_register(self, register):
"""Check if register is a vector register"""
if register is None:
return False
if register['name'].rstrip(string.digits).lower() in ['mm', 'xmm', 'ymm', 'zmm']:
if register["name"].rstrip(string.digits).lower() in ["mm", "xmm", "ymm", "zmm"]:
return True
return False
@@ -443,7 +444,7 @@ class ParserX86ATT(BaseParser):
if register is None:
return False
if self.is_gpr(register):
return 'gpr'
return "gpr"
elif self.is_vector_register(register):
return register['name'].rstrip(string.digits).lower()
return register["name"].rstrip(string.digits).lower()
raise ValueError

View File

@@ -11,13 +11,13 @@ from .marker_utils import reduce_to_section, find_basic_blocks, find_basic_loop_
from .marker_utils import find_jump_labels
__all__ = [
'MachineModel',
'KernelDG',
'reduce_to_section',
'ArchSemantics',
'ISASemantics',
'INSTR_FLAGS',
'find_basic_blocks',
'find_basic_loop_bodies',
'find_jump_labels',
"MachineModel",
"KernelDG",
"reduce_to_section",
"ArchSemantics",
"ISASemantics",
"INSTR_FLAGS",
"find_basic_blocks",
"find_basic_loop_bodies",
"find_jump_labels",
]

View File

@@ -11,7 +11,7 @@ from .isa_semantics import INSTR_FLAGS, ISASemantics
class ArchSemantics(ISASemantics):
GAS_SUFFIXES = 'bswlqt'
GAS_SUFFIXES = "bswlqt"
def __init__(self, machine_model: MachineModel, path_to_yaml=None):
super().__init__(machine_model.get_ISA().lower(), path_to_yaml=path_to_yaml)
@@ -42,15 +42,13 @@ class ArchSemantics(ISASemantics):
kernel.reverse()
port_list = self._machine_model.get_ports()
for instruction_form in kernel:
for uop in instruction_form['port_uops']:
for uop in instruction_form["port_uops"]:
cycles = uop[0]
ports = list(uop[1])
indices = [port_list.index(p) for p in ports]
# check if port sum of used ports for uop are unbalanced
port_sums = self._to_list(itemgetter(*indices)(self.get_throughput_sum(kernel)))
instr_ports = self._to_list(
itemgetter(*indices)(instruction_form['port_pressure'])
)
instr_ports = self._to_list(itemgetter(*indices)(instruction_form["port_pressure"]))
if len(set(port_sums)) > 1:
# balance ports
# init list for keeping track of the current change
@@ -66,7 +64,7 @@ class ArchSemantics(ISASemantics):
differences[max_port_idx] -= INC
differences[min_port_idx] += INC
# instr_ports = [round(p, 2) for p in instr_ports]
self._itemsetter(*indices)(instruction_form['port_pressure'], *instr_ports)
self._itemsetter(*indices)(instruction_form["port_pressure"], *instr_ports)
# check if min port is zero
if round(min(instr_ports), 2) <= 0:
# if port_pressure is not exactly 0.00, add the residual to
@@ -79,20 +77,20 @@ class ArchSemantics(ISASemantics):
# delete it
del differences[instr_ports.index(min(instr_ports))]
self._itemsetter(*indices)(
instruction_form['port_pressure'], *instr_ports
instruction_form["port_pressure"], *instr_ports
)
zero_index = [
p
for p in indices
if round(instruction_form['port_pressure'][p], 2) == 0
if round(instruction_form["port_pressure"][p], 2) == 0
][0]
instruction_form['port_pressure'][zero_index] = 0.0
instruction_form["port_pressure"][zero_index] = 0.0
# Remove from further balancing
indices = [
p for p in indices if instruction_form['port_pressure'][p] > 0
p for p in indices if instruction_form["port_pressure"][p] > 0
]
instr_ports = self._to_list(
itemgetter(*indices)(instruction_form['port_pressure'])
itemgetter(*indices)(instruction_form["port_pressure"])
)
# never remove more than the fixed utilization per uop and port, i.e.,
# cycles/len(ports)
@@ -102,7 +100,7 @@ class ArchSemantics(ISASemantics):
# pressure is not 0
del indices[differences.index(min(differences))]
instr_ports = self._to_list(
itemgetter(*indices)(instruction_form['port_pressure'])
itemgetter(*indices)(instruction_form["port_pressure"])
)
del differences[differences.index(min(differences))]
port_sums = self._to_list(
@@ -112,14 +110,14 @@ class ArchSemantics(ISASemantics):
def set_hidden_loads(self, kernel):
"""Hide loads behind stores if architecture supports hidden loads (depricated)"""
loads = [instr for instr in kernel if INSTR_FLAGS.HAS_LD in instr['flags']]
stores = [instr for instr in kernel if INSTR_FLAGS.HAS_ST in instr['flags']]
loads = [instr for instr in kernel if INSTR_FLAGS.HAS_LD in instr["flags"]]
stores = [instr for instr in kernel if INSTR_FLAGS.HAS_ST in instr["flags"]]
# Filter instructions including load and store
load_ids = [instr['line_number'] for instr in loads]
store_ids = [instr['line_number'] for instr in stores]
load_ids = [instr["line_number"] for instr in loads]
store_ids = [instr["line_number"] for instr in stores]
shared_ldst = list(set(load_ids).intersection(set(store_ids)))
loads = [instr for instr in loads if instr['line_number'] not in shared_ldst]
stores = [instr for instr in stores if instr['line_number'] not in shared_ldst]
loads = [instr for instr in loads if instr["line_number"] not in shared_ldst]
stores = [instr for instr in stores if instr["line_number"] not in shared_ldst]
if len(stores) == 0 or len(loads) == 0:
# nothing to do
@@ -127,53 +125,53 @@ class ArchSemantics(ISASemantics):
if len(loads) <= len(stores):
# Hide all loads
for load in loads:
load['flags'] += [INSTR_FLAGS.HIDDEN_LD]
load['port_pressure'] = self._nullify_data_ports(load['port_pressure'])
load["flags"] += [INSTR_FLAGS.HIDDEN_LD]
load["port_pressure"] = self._nullify_data_ports(load["port_pressure"])
else:
for store in stores:
# Get 'closest' load instruction
min_distance_load = min(
[
(
abs(load_instr['line_number'] - store['line_number']),
load_instr['line_number'],
abs(load_instr["line_number"] - store["line_number"]),
load_instr["line_number"],
)
for load_instr in loads
if INSTR_FLAGS.HIDDEN_LD not in load_instr['flags']
if INSTR_FLAGS.HIDDEN_LD not in load_instr["flags"]
]
)
load = [instr for instr in kernel if instr['line_number'] == min_distance_load[1]][
load = [instr for instr in kernel if instr["line_number"] == min_distance_load[1]][
0
]
# Hide load
load['flags'] += [INSTR_FLAGS.HIDDEN_LD]
load['port_pressure'] = self._nullify_data_ports(load['port_pressure'])
load["flags"] += [INSTR_FLAGS.HIDDEN_LD]
load["port_pressure"] = self._nullify_data_ports(load["port_pressure"])
# get parser result and assign throughput and latency value to instruction form
# mark instruction form with semantic flags
def assign_tp_lt(self, instruction_form):
"""Assign throughput and latency to an instruction form."""
flags = []
port_number = len(self._machine_model['ports'])
if instruction_form['instruction'] is None:
port_number = len(self._machine_model["ports"])
if instruction_form["instruction"] is None:
# No instruction (label, comment, ...) --> ignore
throughput = 0.0
latency = 0.0
latency_wo_load = latency
instruction_form['port_pressure'] = [0.0 for i in range(port_number)]
instruction_form['port_uops'] = []
instruction_form["port_pressure"] = [0.0 for i in range(port_number)]
instruction_form["port_uops"] = []
else:
instruction_data = self._machine_model.get_instruction(
instruction_form['instruction'], instruction_form['operands']
instruction_form["instruction"], instruction_form["operands"]
)
if (
not instruction_data
and self._isa == 'x86'
and instruction_form['instruction'][-1] in self.GAS_SUFFIXES
and self._isa == "x86"
and instruction_form["instruction"][-1] in self.GAS_SUFFIXES
):
# check for instruction without GAS suffix
instruction_data = self._machine_model.get_instruction(
instruction_form['instruction'][:-1], instruction_form['operands']
instruction_form["instruction"][:-1], instruction_form["operands"]
)
if instruction_data:
# instruction form in DB
@@ -190,79 +188,79 @@ class ArchSemantics(ISASemantics):
assign_unknown = True
# check for equivalent register-operands DB entry if LD
if (
INSTR_FLAGS.HAS_LD in instruction_form['flags']
or INSTR_FLAGS.HAS_ST in instruction_form['flags']
INSTR_FLAGS.HAS_LD in instruction_form["flags"]
or INSTR_FLAGS.HAS_ST in instruction_form["flags"]
):
# dynamically combine LD/ST and reg form of instruction form
# substitute mem and look for reg-only variant
operands = self.substitute_mem_address(instruction_form['operands'])
operands = self.substitute_mem_address(instruction_form["operands"])
instruction_data_reg = self._machine_model.get_instruction(
instruction_form['instruction'], operands
instruction_form["instruction"], operands
)
if (
not instruction_data_reg
and self._isa == 'x86'
and instruction_form['instruction'][-1] in self.GAS_SUFFIXES
and self._isa == "x86"
and instruction_form["instruction"][-1] in self.GAS_SUFFIXES
):
# check for instruction without GAS suffix
instruction_data_reg = self._machine_model.get_instruction(
instruction_form['instruction'][:-1], operands
instruction_form["instruction"][:-1], operands
)
if instruction_data_reg:
assign_unknown = False
reg_type = self._parser.get_reg_type(
instruction_data_reg['operands'][
instruction_data_reg["operands"][
operands.index(self._create_reg_wildcard())
]
)
data_port_pressure = [0.0 for _ in range(port_number)]
data_port_uops = []
if INSTR_FLAGS.HAS_LD in instruction_form['flags']:
if INSTR_FLAGS.HAS_LD in instruction_form["flags"]:
# LOAD performance data
data_port_uops = self._machine_model.get_load_throughput(
[
x['memory']
for x in instruction_form['semantic_operands']['source']
+ instruction_form['semantic_operands']['src_dst']
if 'memory' in x
x["memory"]
for x in instruction_form["semantic_operands"]["source"]
+ instruction_form["semantic_operands"]["src_dst"]
if "memory" in x
][0]
)
data_port_pressure = self._machine_model.average_port_pressure(
data_port_uops
)
if 'load_throughput_multiplier' in self._machine_model:
multiplier = self._machine_model['load_throughput_multiplier'][
if "load_throughput_multiplier" in self._machine_model:
multiplier = self._machine_model["load_throughput_multiplier"][
reg_type
]
data_port_pressure = [pp * multiplier for pp in data_port_pressure]
if INSTR_FLAGS.HAS_ST in instruction_form['flags']:
if INSTR_FLAGS.HAS_ST in instruction_form["flags"]:
# STORE performance data
destinations = (
instruction_form['semantic_operands']['destination']
+ instruction_form['semantic_operands']['src_dst']
instruction_form["semantic_operands"]["destination"]
+ instruction_form["semantic_operands"]["src_dst"]
)
st_data_port_uops = self._machine_model.get_store_throughput(
[x['memory'] for x in destinations if 'memory' in x][0]
[x["memory"] for x in destinations if "memory" in x][0]
)
# zero data port pressure and remove HAS_ST flag if
# - no mem operand in dst &&
# - all mem operands in src_dst are pre-/post-indexed
# since it is no mem store
if (
self._isa == 'aarch64'
and 'memory'
not in instruction_form['semantic_operands']['destination']
self._isa == "aarch64"
and "memory"
not in instruction_form["semantic_operands"]["destination"]
and all(
[
'post_indexed' in op['memory']
or 'pre_indexed' in op['memory']
for op in instruction_form['semantic_operands']['src_dst']
if 'memory' in op
"post_indexed" in op["memory"]
or "pre_indexed" in op["memory"]
for op in instruction_form["semantic_operands"]["src_dst"]
if "memory" in op
]
)
):
st_data_port_uops = []
instruction_form['flags'].remove(INSTR_FLAGS.HAS_ST)
instruction_form["flags"].remove(INSTR_FLAGS.HAS_ST)
# sum up all data ports in case for LOAD and STORE
st_data_port_pressure = self._machine_model.average_port_pressure(
@@ -273,21 +271,21 @@ class ArchSemantics(ISASemantics):
]
data_port_uops += st_data_port_uops
throughput = max(
max(data_port_pressure), instruction_data_reg['throughput']
max(data_port_pressure), instruction_data_reg["throughput"]
)
latency = instruction_data_reg['latency']
latency = instruction_data_reg["latency"]
# Add LD and ST latency
latency += (
self._machine_model.get_load_latency(reg_type)
if INSTR_FLAGS.HAS_LD in instruction_form['flags']
if INSTR_FLAGS.HAS_LD in instruction_form["flags"]
else 0
)
latency += (
self._machine_model.get_store_latency(reg_type)
if INSTR_FLAGS.HAS_ST in instruction_form['flags']
if INSTR_FLAGS.HAS_ST in instruction_form["flags"]
else 0
)
latency_wo_load = instruction_data_reg['latency']
latency_wo_load = instruction_data_reg["latency"]
# add latency of ADD if post- or pre-indexed load
# TODO more investigation: check dot-graph, wrong latency distribution!
# if (
@@ -302,17 +300,17 @@ class ArchSemantics(ISASemantics):
# )
# ):
# latency_wo_load = 1.0
instruction_form['port_pressure'] = [
instruction_form["port_pressure"] = [
sum(x)
for x in zip(
data_port_pressure,
self._machine_model.average_port_pressure(
instruction_data_reg['port_pressure']
instruction_data_reg["port_pressure"]
),
)
]
instruction_form['port_uops'] = list(
chain(instruction_data_reg['port_pressure'], data_port_uops)
instruction_form["port_uops"] = list(
chain(instruction_data_reg["port_pressure"], data_port_uops)
)
if assign_unknown:
@@ -320,68 +318,66 @@ class ArchSemantics(ISASemantics):
throughput = 0.0
latency = 0.0
latency_wo_load = latency
instruction_form['port_pressure'] = [0.0 for i in range(port_number)]
instruction_form['port_uops'] = []
instruction_form["port_pressure"] = [0.0 for i in range(port_number)]
instruction_form["port_uops"] = []
flags += [INSTR_FLAGS.TP_UNKWN, INSTR_FLAGS.LT_UNKWN]
# flatten flag list
flags = list(set(flags))
if 'flags' not in instruction_form:
instruction_form['flags'] = flags
if "flags" not in instruction_form:
instruction_form["flags"] = flags
else:
instruction_form['flags'] += flags
instruction_form['throughput'] = throughput
instruction_form['latency'] = latency
instruction_form['latency_wo_load'] = latency_wo_load
instruction_form["flags"] += flags
instruction_form["throughput"] = throughput
instruction_form["latency"] = latency
instruction_form["latency_wo_load"] = latency_wo_load
# for later CP and loop-carried dependency analysis
instruction_form['latency_cp'] = 0
instruction_form['latency_lcd'] = 0
instruction_form["latency_cp"] = 0
instruction_form["latency_lcd"] = 0
def _handle_instruction_found(self, instruction_data, port_number, instruction_form, flags):
"""Apply performance data to instruction if it was found in the archDB"""
throughput = instruction_data['throughput']
port_pressure = self._machine_model.average_port_pressure(
instruction_data['port_pressure']
)
instruction_form['port_uops'] = instruction_data['port_pressure']
throughput = instruction_data["throughput"]
port_pressure = self._machine_model.average_port_pressure(instruction_data["port_pressure"])
instruction_form["port_uops"] = instruction_data["port_pressure"]
try:
assert isinstance(port_pressure, list)
assert len(port_pressure) == port_number
instruction_form['port_pressure'] = port_pressure
instruction_form["port_pressure"] = port_pressure
if sum(port_pressure) == 0 and throughput is not None:
# port pressure on all ports 0 --> not bound to a port
flags.append(INSTR_FLAGS.NOT_BOUND)
except AssertionError:
warnings.warn(
'Port pressure could not be imported correctly from database. '
+ 'Please check entry for:\n {}'.format(instruction_form)
"Port pressure could not be imported correctly from database. "
+ "Please check entry for:\n {}".format(instruction_form)
)
instruction_form['port_pressure'] = [0.0 for i in range(port_number)]
instruction_form['port_uops'] = []
instruction_form["port_pressure"] = [0.0 for i in range(port_number)]
instruction_form["port_uops"] = []
flags.append(INSTR_FLAGS.TP_UNKWN)
if throughput is None:
# assume 0 cy and mark as unknown
throughput = 0.0
flags.append(INSTR_FLAGS.TP_UNKWN)
latency = instruction_data['latency']
latency = instruction_data["latency"]
latency_wo_load = latency
if latency is None:
# assume 0 cy and mark as unknown
latency = 0.0
latency_wo_load = latency
flags.append(INSTR_FLAGS.LT_UNKWN)
if INSTR_FLAGS.HAS_LD in instruction_form['flags']:
if INSTR_FLAGS.HAS_LD in instruction_form["flags"]:
flags.append(INSTR_FLAGS.LD)
return throughput, port_pressure, latency, latency_wo_load
def convert_op_to_reg(self, reg_type, reg_id='0'):
def convert_op_to_reg(self, reg_type, reg_id="0"):
"""Create register operand for a memory addressing operand"""
if self._isa == 'x86':
if reg_type == 'gpr':
register = {'register': {'name': 'r' + str(int(reg_id) + 9)}}
if self._isa == "x86":
if reg_type == "gpr":
register = {"register": {"name": "r" + str(int(reg_id) + 9)}}
else:
register = {'register': {'name': reg_type + reg_id}}
elif self._isa == 'aarch64':
register = {'register': {'prefix': reg_type, 'name': reg_id}}
register = {"register": {"name": reg_type + reg_id}}
elif self._isa == "aarch64":
register = {"register": {"prefix": reg_type, "name": reg_id}}
return register
def _nullify_data_ports(self, port_pressure):
@@ -398,7 +394,9 @@ class ArchSemantics(ISASemantics):
def g(obj, value):
obj[item] = value
else:
def g(obj, *values):
for item, value in zip(items, values):
obj[item] = value
@@ -416,7 +414,7 @@ class ArchSemantics(ISASemantics):
"""Get the overall throughput sum separated by port of all instructions of a kernel."""
# ignoring all lines with throughput == 0.0, because there won't be anything to sum up
# typically comment, label and non-instruction lines
port_pressures = [instr['port_pressure'] for instr in kernel if instr['throughput'] != 0.0]
port_pressures = [instr["port_pressure"] for instr in kernel if instr["throughput"] != 0.0]
# Essentially summing up each columns of port_pressures, where each column is one port
# and each row is one line of the kernel
# round is necessary to ensure termination of ArchsSemantics.assign_optimal_throughput

View File

@@ -18,76 +18,77 @@ from osaca.parser import ParserX86ATT
class MachineModel(object):
WILDCARD = '*'
WILDCARD = "*"
INTERNAL_VERSION = 1 # increase whenever self._data format changes to invalidate cache!
def __init__(self, arch=None, path_to_yaml=None, isa=None, lazy=False):
if not arch and not path_to_yaml:
if not isa:
raise ValueError('One of arch, path_to_yaml and isa must be specified')
raise ValueError("One of arch, path_to_yaml and isa must be specified")
self._data = {
'osaca_version': str(__version__),
'micro_architecture': None,
'arch_code': None,
'isa': isa,
'ROB_size': None,
'retired_uOps_per_cycle': None,
'scheduler_size': None,
'hidden_loads': None,
'load_latency': {},
'load_throughput': [
{'base': b, 'index': i, 'offset': o, 'scale': s, 'port_pressure': []}
for b, i, o, s in product(['gpr'], ['gpr', None], ['imd', None], [1, 8])
"osaca_version": str(__version__),
"micro_architecture": None,
"arch_code": None,
"isa": isa,
"ROB_size": None,
"retired_uOps_per_cycle": None,
"scheduler_size": None,
"hidden_loads": None,
"load_latency": {},
"load_throughput": [
{"base": b, "index": i, "offset": o, "scale": s, "port_pressure": []}
for b, i, o, s in product(["gpr"], ["gpr", None], ["imd", None], [1, 8])
],
'load_throughput_default': [],
'store_throughput': [],
'store_throughput_default': [],
'ports': [],
'port_model_scheme': None,
'instruction_forms': []
"load_throughput_default": [],
"store_throughput": [],
"store_throughput_default": [],
"ports": [],
"port_model_scheme": None,
"instruction_forms": [],
}
else:
if arch and path_to_yaml:
raise ValueError('Only one of arch and path_to_yaml is allowed.')
raise ValueError("Only one of arch and path_to_yaml is allowed.")
self._path = path_to_yaml
self._arch = arch
yaml = self._create_yaml_object()
if arch:
self._arch = arch.lower()
self._path = utils.find_datafile(self._arch + '.yml')
self._path = utils.find_datafile(self._arch + ".yml")
# check if file is cached
cached = self._get_cached(self._path) if not lazy else False
if cached:
self._data = cached
else:
# otherwise load
with open(self._path, 'r') as f:
with open(self._path, "r") as f:
if not lazy:
self._data = yaml.load(f)
else:
file_content = ''
file_content = ""
line = f.readline()
while 'instruction_forms:' not in line:
while "instruction_forms:" not in line:
file_content += line
line = f.readline()
self._data = yaml.load(file_content)
self._data['instruction_forms'] = []
self._data["instruction_forms"] = []
# separate multi-alias instruction forms
for entry in [x for x in self._data['instruction_forms']
if isinstance(x['name'], list)]:
for name in entry['name']:
new_entry = {'name': name}
for k in [x for x in entry.keys() if x != 'name']:
for entry in [
x for x in self._data["instruction_forms"] if isinstance(x["name"], list)
]:
for name in entry["name"]:
new_entry = {"name": name}
for k in [x for x in entry.keys() if x != "name"]:
new_entry[k] = entry[k]
self._data['instruction_forms'].append(new_entry)
self._data["instruction_forms"].append(new_entry)
# remove old entry
self._data['instruction_forms'].remove(entry)
self._data["instruction_forms"].remove(entry)
# Normalize instruction_form names (to UPPERCASE) and build dict for faster access:
self._data['instruction_forms_dict'] = defaultdict(list)
for iform in self._data['instruction_forms']:
iform['name'] = iform['name'].upper()
self._data['instruction_forms_dict'][iform['name']].append(iform)
self._data['internal_version'] = self.INTERNAL_VERSION
self._data["instruction_forms_dict"] = defaultdict(list)
for iform in self._data["instruction_forms"]:
iform["name"] = iform["name"].upper()
self._data["instruction_forms_dict"][iform["name"]].append(iform)
self._data["internal_version"] = self.INTERNAL_VERSION
if not lazy:
# cache internal representation for future use
@@ -108,22 +109,24 @@ class MachineModel(object):
# For use with dict instead of list as DB
if name is None:
return None
name_matched_iforms = self._data['instruction_forms_dict'].get(name.upper(), [])
name_matched_iforms = self._data["instruction_forms_dict"].get(name.upper(), [])
try:
return next(
instruction_form
for instruction_form in name_matched_iforms if self._match_operands(
instruction_form['operands'] if 'operands' in instruction_form else [],
operands))
for instruction_form in name_matched_iforms
if self._match_operands(
instruction_form["operands"] if "operands" in instruction_form else [], operands
)
)
except StopIteration:
return None
except TypeError as e:
print('\nname: {}\noperands: {}'.format(name, operands))
print("\nname: {}\noperands: {}".format(name, operands))
raise TypeError from e
def average_port_pressure(self, port_pressure):
"""Construct average port pressure list from instruction data."""
port_list = self._data['ports']
port_list = self._data["ports"]
average_pressure = [0.0] * len(port_list)
for cycles, ports in port_pressure:
for p in ports:
@@ -138,59 +141,59 @@ class MachineModel(object):
instr_data = self.get_instruction(name, operands)
if instr_data is None:
instr_data = {}
self._data['instruction_forms'].append(instr_data)
self._data["instruction_forms"].append(instr_data)
instr_data['name'] = name
instr_data['operands'] = operands
instr_data['latency'] = latency
instr_data['port_pressure'] = port_pressure
instr_data['throughput'] = throughput
instr_data['uops'] = uops
instr_data["name"] = name
instr_data["operands"] = operands
instr_data["latency"] = latency
instr_data["port_pressure"] = port_pressure
instr_data["throughput"] = throughput
instr_data["uops"] = uops
def set_instruction_entry(self, entry):
"""Import instruction as entry object form information."""
self.set_instruction(
entry['name'],
entry['operands'] if 'operands' in entry else None,
entry['latency'] if 'latency' in entry else None,
entry['port_pressure'] if 'port_pressure' in entry else None,
entry['throughput'] if 'throughput' in entry else None,
entry['uops'] if 'uops' in entry else None,
entry["name"],
entry["operands"] if "operands" in entry else None,
entry["latency"] if "latency" in entry else None,
entry["port_pressure"] if "port_pressure" in entry else None,
entry["throughput"] if "throughput" in entry else None,
entry["uops"] if "uops" in entry else None,
)
def add_port(self, port):
"""Add port in port model of current machine model."""
if port not in self._data['ports']:
self._data['ports'].append(port)
if port not in self._data["ports"]:
self._data["ports"].append(port)
def get_ISA(self):
"""Return ISA of :class:`MachineModel`."""
return self._data['isa'].lower()
return self._data["isa"].lower()
def get_arch(self):
"""Return micro-architecture code of :class:`MachineModel`."""
return self._data['arch_code'].lower()
return self._data["arch_code"].lower()
def get_ports(self):
"""Return port model of :class:`MachineModel`."""
return self._data['ports']
return self._data["ports"]
def has_hidden_loads(self):
"""Return if model has hidden loads."""
if 'hidden_loads' in self._data:
return self._data['hidden_loads']
if "hidden_loads" in self._data:
return self._data["hidden_loads"]
return False
def get_load_latency(self, reg_type):
"""Return load latency for given register type."""
return self._data['load_latency'][reg_type] if self._data['load_latency'][reg_type] else 0
return self._data["load_latency"][reg_type] if self._data["load_latency"][reg_type] else 0
def get_load_throughput(self, memory):
"""Return load thorughput for given register type."""
ld_tp = [m for m in self._data['load_throughput'] if self._match_mem_entries(memory, m)]
ld_tp = [m for m in self._data["load_throughput"] if self._match_mem_entries(memory, m)]
if len(ld_tp) > 0:
return ld_tp[0]['port_pressure'].copy()
return self._data['load_throughput_default'].copy()
return ld_tp[0]["port_pressure"].copy()
return self._data["load_throughput_default"].copy()
def get_store_latency(self, reg_type):
"""Return store latency for given register type."""
@@ -199,61 +202,61 @@ class MachineModel(object):
def get_store_throughput(self, memory):
"""Return store throughput for given register type."""
st_tp = [m for m in self._data['store_throughput'] if self._match_mem_entries(memory, m)]
st_tp = [m for m in self._data["store_throughput"] if self._match_mem_entries(memory, m)]
if len(st_tp) > 0:
return st_tp[0]['port_pressure'].copy()
return self._data['store_throughput_default'].copy()
return st_tp[0]["port_pressure"].copy()
return self._data["store_throughput_default"].copy()
def _match_mem_entries(self, mem, i_mem):
"""Check if memory addressing ``mem`` and ``i_mem`` are of the same type."""
if self._data['isa'].lower() == 'aarch64':
if self._data["isa"].lower() == "aarch64":
return self._is_AArch64_mem_type(i_mem, mem)
if self._data['isa'].lower() == 'x86':
if self._data["isa"].lower() == "x86":
return self._is_x86_mem_type(i_mem, mem)
def get_data_ports(self):
"""Return all data ports (i.e., ports with D-suffix) of current model."""
data_port = re.compile(r'^[0-9]+D$')
data_ports = [x for x in filter(data_port.match, self._data['ports'])]
data_port = re.compile(r"^[0-9]+D$")
data_ports = [x for x in filter(data_port.match, self._data["ports"])]
return data_ports
@staticmethod
def get_full_instruction_name(instruction_form):
"""Get one instruction name string including the mnemonic and all operands."""
operands = []
for op in instruction_form['operands']:
for op in instruction_form["operands"]:
op_attrs = [
y + ':' + str(op[y])
for y in list(filter(lambda x: True if x != 'class' else False, op))
y + ":" + str(op[y])
for y in list(filter(lambda x: True if x != "class" else False, op))
]
operands.append('{}({})'.format(op['class'], ','.join(op_attrs)))
return '{} {}'.format(instruction_form['name'].lower(), ','.join(operands))
operands.append("{}({})".format(op["class"], ",".join(op_attrs)))
return "{} {}".format(instruction_form["name"].lower(), ",".join(operands))
@staticmethod
def get_isa_for_arch(arch):
"""Return ISA for given micro-arch ``arch``."""
arch_dict = {
'a64fx': 'aarch64',
'tx2': 'aarch64',
'n1': 'aarch64',
'zen1': 'x86',
'zen+': 'x86',
'zen2': 'x86',
'con': 'x86', # Intel Conroe
'wol': 'x86', # Intel Wolfdale
'snb': 'x86',
'ivb': 'x86',
'hsw': 'x86',
'bdw': 'x86',
'skl': 'x86',
'skx': 'x86',
'csx': 'x86',
'wsm': 'x86',
'nhm': 'x86',
'kbl': 'x86',
'cnl': 'x86',
'cfl': 'x86',
'icl': 'x86',
"a64fx": "aarch64",
"tx2": "aarch64",
"n1": "aarch64",
"zen1": "x86",
"zen+": "x86",
"zen2": "x86",
"con": "x86", # Intel Conroe
"wol": "x86", # Intel Wolfdale
"snb": "x86",
"ivb": "x86",
"hsw": "x86",
"bdw": "x86",
"skl": "x86",
"skx": "x86",
"csx": "x86",
"wsm": "x86",
"nhm": "x86",
"kbl": "x86",
"cnl": "x86",
"cfl": "x86",
"icl": "x86",
}
arch = arch.lower()
if arch in arch_dict:
@@ -264,16 +267,16 @@ class MachineModel(object):
def dump(self, stream=None):
"""Dump machine model to stream or return it as a ``str`` if no stream is given."""
# Replace instruction form's port_pressure with styled version for RoundtripDumper
formatted_instruction_forms = deepcopy(self._data['instruction_forms'])
formatted_instruction_forms = deepcopy(self._data["instruction_forms"])
for instruction_form in formatted_instruction_forms:
if instruction_form['port_pressure'] is not None:
cs = ruamel.yaml.comments.CommentedSeq(instruction_form['port_pressure'])
if instruction_form["port_pressure"] is not None:
cs = ruamel.yaml.comments.CommentedSeq(instruction_form["port_pressure"])
cs.fa.set_flow_style()
instruction_form['port_pressure'] = cs
instruction_form["port_pressure"] = cs
# Replace load_throughput with styled version for RoundtripDumper
formatted_load_throughput = []
for lt in self._data['load_throughput']:
for lt in self._data["load_throughput"]:
cm = ruamel.yaml.comments.CommentedMap(lt)
cm.fa.set_flow_style()
formatted_load_throughput.append(cm)
@@ -287,13 +290,18 @@ class MachineModel(object):
{
k: v
for k, v in self._data.items()
if k not in ['instruction_forms', 'instruction_forms_dict', 'load_throughput',
'internal_version']
if k
not in [
"instruction_forms",
"instruction_forms_dict",
"load_throughput",
"internal_version",
]
},
stream,
)
yaml.dump({'load_throughput': formatted_load_throughput}, stream)
yaml.dump({'instruction_forms': formatted_instruction_forms}, stream)
yaml.dump({"load_throughput": formatted_load_throughput}, stream)
yaml.dump({"instruction_forms": formatted_instruction_forms}, stream)
if isinstance(stream, StringIO):
return stream.getvalue()
@@ -312,21 +320,21 @@ class MachineModel(object):
hexhash = hashlib.sha256(p.read_bytes()).hexdigest()
# 1. companion cachefile: same location, with '.<name>_<sha512hash>.pickle'
companion_cachefile = p.with_name('.' + p.stem + '_' + hexhash).with_suffix('.pickle')
companion_cachefile = p.with_name("." + p.stem + "_" + hexhash).with_suffix(".pickle")
if companion_cachefile.exists():
# companion file (must be up-to-date, due to equal hash)
with companion_cachefile.open('rb') as f:
with companion_cachefile.open("rb") as f:
data = pickle.load(f)
if data.get('internal_version') == self.INTERNAL_VERSION:
if data.get("internal_version") == self.INTERNAL_VERSION:
return data
# 2. home cachefile: ~/.osaca/cache/<name>_<sha512hash>.pickle
home_cachefile = (Path(utils.CACHE_DIR) / (p.stem + '_' + hexhash)).with_suffix('.pickle')
home_cachefile = (Path(utils.CACHE_DIR) / (p.stem + "_" + hexhash)).with_suffix(".pickle")
if home_cachefile.exists():
# home file (must be up-to-date, due to equal hash)
with home_cachefile.open('rb') as f:
with home_cachefile.open("rb") as f:
data = pickle.load(f)
if data.get('internal_version') == self.INTERNAL_VERSION:
if data.get("internal_version") == self.INTERNAL_VERSION:
return data
return False
@@ -340,9 +348,9 @@ class MachineModel(object):
p = Path(filepath)
hexhash = hashlib.sha256(p.read_bytes()).hexdigest()
# 1. companion cachefile: same location, with '.<name>_<sha512hash>.pickle'
companion_cachefile = p.with_name('.' + p.stem + '_' + hexhash).with_suffix('.pickle')
companion_cachefile = p.with_name("." + p.stem + "_" + hexhash).with_suffix(".pickle")
if os.access(str(companion_cachefile.parent), os.W_OK):
with companion_cachefile.open('wb') as f:
with companion_cachefile.open("wb") as f:
pickle.dump(self._data, f)
return
@@ -352,92 +360,92 @@ class MachineModel(object):
os.makedirs(cache_dir, exist_ok=True)
except OSError:
return
home_cachefile = (cache_dir / (p.stem + '_' + hexhash)).with_suffix('.pickle')
home_cachefile = (cache_dir / (p.stem + "_" + hexhash)).with_suffix(".pickle")
if os.access(str(home_cachefile.parent), os.W_OK):
with home_cachefile.open('wb') as f:
with home_cachefile.open("wb") as f:
pickle.dump(self._data, f)
def _get_key(self, name, operands):
"""Get unique instruction form key for dict DB."""
key_string = name.lower() + '-'
key_string = name.lower() + "-"
if operands is None:
return key_string[:-1]
key_string += '_'.join([self._get_operand_hash(op) for op in operands])
key_string += "_".join([self._get_operand_hash(op) for op in operands])
return key_string
def _get_operand_hash(self, operand):
"""Get unique key for operand for dict DB"""
operand_string = ''
if 'class' in operand:
operand_string = ""
if "class" in operand:
# DB entry
opclass = operand['class']
opclass = operand["class"]
else:
# parsed instruction
opclass = list(operand.keys())[0]
operand = operand[opclass]
if opclass == 'immediate':
if opclass == "immediate":
# Immediate
operand_string += 'i'
elif opclass == 'register':
operand_string += "i"
elif opclass == "register":
# Register
if 'prefix' in operand:
operand_string += operand['prefix']
operand_string += operand['shape'] if 'shape' in operand else ''
elif 'name' in operand:
operand_string += 'r' if operand['name'] == 'gpr' else operand['name'][0]
elif opclass == 'memory':
if "prefix" in operand:
operand_string += operand["prefix"]
operand_string += operand["shape"] if "shape" in operand else ""
elif "name" in operand:
operand_string += "r" if operand["name"] == "gpr" else operand["name"][0]
elif opclass == "memory":
# Memory
operand_string += 'm'
operand_string += 'b' if operand['base'] is not None else ''
operand_string += 'o' if operand['offset'] is not None else ''
operand_string += 'i' if operand['index'] is not None else ''
operand_string += "m"
operand_string += "b" if operand["base"] is not None else ""
operand_string += "o" if operand["offset"] is not None else ""
operand_string += "i" if operand["index"] is not None else ""
operand_string += (
's' if operand['scale'] == self.WILDCARD or operand['scale'] > 1 else ''
"s" if operand["scale"] == self.WILDCARD or operand["scale"] > 1 else ""
)
if 'pre-indexed' in operand:
operand_string += 'r' if operand['pre-indexed'] else ''
operand_string += 'p' if operand['post-indexed'] else ''
if "pre-indexed" in operand:
operand_string += "r" if operand["pre-indexed"] else ""
operand_string += "p" if operand["post-indexed"] else ""
return operand_string
def _create_db_operand_aarch64(self, operand):
"""Create instruction form operand for DB out of operand string."""
if operand == 'i':
return {'class': 'immediate', 'imd': 'int'}
elif operand in 'wxbhsdq':
return {'class': 'register', 'prefix': operand}
elif operand.startswith('v'):
return {'class': 'register', 'prefix': 'v', 'shape': operand[1:2]}
elif operand.startswith('m'):
if operand == "i":
return {"class": "immediate", "imd": "int"}
elif operand in "wxbhsdq":
return {"class": "register", "prefix": operand}
elif operand.startswith("v"):
return {"class": "register", "prefix": "v", "shape": operand[1:2]}
elif operand.startswith("m"):
return {
'class': 'memory',
'base': 'x' if 'b' in operand else None,
'offset': 'imd' if 'o' in operand else None,
'index': 'gpr' if 'i' in operand else None,
'scale': 8 if 's' in operand else 1,
'pre-indexed': True if 'r' in operand else False,
'post-indexed': True if 'p' in operand else False,
"class": "memory",
"base": "x" if "b" in operand else None,
"offset": "imd" if "o" in operand else None,
"index": "gpr" if "i" in operand else None,
"scale": 8 if "s" in operand else 1,
"pre-indexed": True if "r" in operand else False,
"post-indexed": True if "p" in operand else False,
}
else:
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
raise ValueError("Parameter {} is not a valid operand code".format(operand))
def _create_db_operand_x86(self, operand):
"""Create instruction form operand for DB out of operand string."""
if operand == 'r':
return {'class': 'register', 'name': 'gpr'}
elif operand in 'xyz':
return {'class': 'register', 'name': operand + 'mm'}
elif operand == 'i':
return {'class': 'immediate', 'imd': 'int'}
elif operand.startswith('m'):
if operand == "r":
return {"class": "register", "name": "gpr"}
elif operand in "xyz":
return {"class": "register", "name": operand + "mm"}
elif operand == "i":
return {"class": "immediate", "imd": "int"}
elif operand.startswith("m"):
return {
'class': 'memory',
'base': 'gpr' if 'b' in operand else None,
'offset': 'imd' if 'o' in operand else None,
'index': 'gpr' if 'i' in operand else None,
'scale': 8 if 's' in operand else 1,
"class": "memory",
"base": "gpr" if "b" in operand else None,
"offset": "imd" if "o" in operand else None,
"index": "gpr" if "i" in operand else None,
"scale": 8 if "s" in operand else 1,
}
else:
raise ValueError('Parameter {} is not a valid operand code'.format(operand))
raise ValueError("Parameter {} is not a valid operand code".format(operand))
def _check_for_duplicate(self, name, operands):
"""
@@ -450,9 +458,9 @@ class MachineModel(object):
"""
matches = [
instruction_form
for instruction_form in self._data['instruction_forms']
if instruction_form['name'].lower() == name.lower()
and self._match_operands(instruction_form['operands'], operands)
for instruction_form in self._data["instruction_forms"]
if instruction_form["name"].lower() == name.lower()
and self._match_operands(instruction_form["operands"], operands)
]
if len(matches) > 1:
return True
@@ -475,80 +483,76 @@ class MachineModel(object):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
# check for wildcard
if self.WILDCARD in operand:
if (
'class' in i_operand
and i_operand['class'] == 'register'
or 'register' in i_operand
):
if "class" in i_operand and i_operand["class"] == "register" or "register" in i_operand:
return True
else:
return False
if self._data['isa'].lower() == 'aarch64':
if self._data["isa"].lower() == "aarch64":
return self._check_AArch64_operands(i_operand, operand)
if self._data['isa'].lower() == 'x86':
if self._data["isa"].lower() == "x86":
return self._check_x86_operands(i_operand, operand)
def _check_AArch64_operands(self, i_operand, operand):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
if 'class' in operand:
if "class" in operand:
# compare two DB entries
return self._compare_db_entries(i_operand, operand)
# TODO support class wildcards
# register
if 'register' in operand:
if i_operand['class'] != 'register':
if "register" in operand:
if i_operand["class"] != "register":
return False
return self._is_AArch64_reg_type(i_operand, operand['register'])
return self._is_AArch64_reg_type(i_operand, operand["register"])
# memory
if 'memory' in operand:
if i_operand['class'] != 'memory':
if "memory" in operand:
if i_operand["class"] != "memory":
return False
return self._is_AArch64_mem_type(i_operand, operand['memory'])
return self._is_AArch64_mem_type(i_operand, operand["memory"])
# immediate
# TODO support wildcards
if 'value' in operand or ('immediate' in operand and 'value' in operand['immediate']):
return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int'
if 'float' in operand or ('immediate' in operand and 'float' in operand['immediate']):
return i_operand['class'] == 'immediate' and i_operand['imd'] == 'float'
if 'double' in operand or ('immediate' in operand and 'double' in operand['immediate']):
return i_operand['class'] == 'immediate' and i_operand['imd'] == 'double'
if "value" in operand or ("immediate" in operand and "value" in operand["immediate"]):
return i_operand["class"] == "immediate" and i_operand["imd"] == "int"
if "float" in operand or ("immediate" in operand and "float" in operand["immediate"]):
return i_operand["class"] == "immediate" and i_operand["imd"] == "float"
if "double" in operand or ("immediate" in operand and "double" in operand["immediate"]):
return i_operand["class"] == "immediate" and i_operand["imd"] == "double"
# identifier
if 'identifier' in operand or (
'immediate' in operand and 'identifier' in operand['immediate']
if "identifier" in operand or (
"immediate" in operand and "identifier" in operand["immediate"]
):
return i_operand['class'] == 'identifier'
return i_operand["class"] == "identifier"
# prefetch option
if 'prfop' in operand:
return i_operand['class'] == 'prfop'
if "prfop" in operand:
return i_operand["class"] == "prfop"
# no match
return False
def _check_x86_operands(self, i_operand, operand):
"""Check if the types of operand ``i_operand`` and ``operand`` match."""
if 'class' in operand:
if "class" in operand:
# compare two DB entries
return self._compare_db_entries(i_operand, operand)
# register
if 'register' in operand:
if i_operand['class'] != 'register':
if "register" in operand:
if i_operand["class"] != "register":
return False
return self._is_x86_reg_type(i_operand, operand['register'], consider_masking=True)
return self._is_x86_reg_type(i_operand, operand["register"], consider_masking=True)
# memory
if 'memory' in operand:
if i_operand['class'] != 'memory':
if "memory" in operand:
if i_operand["class"] != "memory":
return False
return self._is_x86_mem_type(i_operand, operand['memory'])
return self._is_x86_mem_type(i_operand, operand["memory"])
# immediate
if 'immediate' in operand or 'value' in operand:
return i_operand['class'] == 'immediate' and i_operand['imd'] == 'int'
if "immediate" in operand or "value" in operand:
return i_operand["class"] == "immediate" and i_operand["imd"] == "int"
# identifier (e.g., labels)
if 'identifier' in operand:
return i_operand['class'] == 'identifier'
if "identifier" in operand:
return i_operand["class"] == "identifier"
def _compare_db_entries(self, operand_1, operand_2):
"""Check if operand types in DB format (i.e., not parsed) match."""
operand_attributes = list(
filter(lambda x: True if x != 'source' and x != 'destination' else False, operand_1)
filter(lambda x: True if x != "source" and x != "destination" else False, operand_1)
)
for key in operand_attributes:
try:
@@ -563,22 +567,21 @@ class MachineModel(object):
def _is_AArch64_reg_type(self, i_reg, reg):
"""Check if register type match."""
# check for wildcards
if reg['prefix'] == self.WILDCARD or i_reg['prefix'] == self.WILDCARD:
if 'shape' in reg:
if 'shape' in i_reg and (
reg['shape'] == i_reg['shape']
or self.WILDCARD in (reg['shape'] + i_reg['shape'])
if reg["prefix"] == self.WILDCARD or i_reg["prefix"] == self.WILDCARD:
if "shape" in reg:
if "shape" in i_reg and (
reg["shape"] == i_reg["shape"]
or self.WILDCARD in (reg["shape"] + i_reg["shape"])
):
return True
return False
return True
# check for prefix and shape
if reg['prefix'] != i_reg['prefix']:
if reg["prefix"] != i_reg["prefix"]:
return False
if 'shape' in reg:
if 'shape' in i_reg and (
reg['shape'] == i_reg['shape']
or self.WILDCARD in (reg['shape'] + i_reg['shape'])
if "shape" in reg:
if "shape" in i_reg and (
reg["shape"] == i_reg["shape"] or self.WILDCARD in (reg["shape"] + i_reg["shape"])
):
return True
return False
@@ -586,48 +589,48 @@ class MachineModel(object):
def _is_x86_reg_type(self, i_reg, reg, consider_masking=False):
"""Check if register type match."""
i_reg_name = i_reg['name'] if i_reg and 'name' in i_reg else i_reg
i_reg_name = i_reg["name"] if i_reg and "name" in i_reg else i_reg
if reg is None:
if i_reg is None:
return True
return False
# check for wildcards
if i_reg_name == self.WILDCARD or reg['name'] == self.WILDCARD:
if i_reg_name == self.WILDCARD or reg["name"] == self.WILDCARD:
return True
# differentiate between vector registers (mm, xmm, ymm, zmm) and others (gpr)
parser_x86 = ParserX86ATT()
if parser_x86.is_vector_register(reg):
if reg['name'].rstrip(string.digits).lower() == i_reg_name:
if reg["name"].rstrip(string.digits).lower() == i_reg_name:
# Consider masking and zeroing for AVX512
if consider_masking:
mask_ok = zero_ok = True
if 'mask' in reg or 'mask' in i_reg:
if "mask" in reg or "mask" in i_reg:
# one instruction is missing the masking while the other has it
mask_ok = False
# check for wildcard
if (
(
'mask' in reg
and reg['mask'].rstrip(string.digits).lower() == i_reg.get('mask')
"mask" in reg
and reg["mask"].rstrip(string.digits).lower() == i_reg.get("mask")
)
or reg.get('mask') == self.WILDCARD
or i_reg.get('mask') == self.WILDCARD
or reg.get("mask") == self.WILDCARD
or i_reg.get("mask") == self.WILDCARD
):
mask_ok = True
if bool('zeroing' in reg) ^ bool('zeroing' in i_reg):
if bool("zeroing" in reg) ^ bool("zeroing" in i_reg):
# one instruction is missing zeroing while the other has it
zero_ok = False
# check for wildcard
if (
i_reg.get('zeroing') == self.WILDCARD
or reg.get('zeroing') == self.WILDCARD
i_reg.get("zeroing") == self.WILDCARD
or reg.get("zeroing") == self.WILDCARD
):
zero_ok = True
if not mask_ok or not zero_ok:
return False
return True
else:
if i_reg_name == 'gpr':
if i_reg_name == "gpr":
return True
return False
@@ -636,50 +639,50 @@ class MachineModel(object):
if (
# check base
(
(mem['base'] is None and i_mem['base'] is None)
or i_mem['base'] == self.WILDCARD
or mem['base']['prefix'] == i_mem['base']
(mem["base"] is None and i_mem["base"] is None)
or i_mem["base"] == self.WILDCARD
or mem["base"]["prefix"] == i_mem["base"]
)
# check offset
and (
mem['offset'] == i_mem['offset']
or i_mem['offset'] == self.WILDCARD
mem["offset"] == i_mem["offset"]
or i_mem["offset"] == self.WILDCARD
or (
mem['offset'] is not None
and 'identifier' in mem['offset']
and i_mem['offset'] == 'identifier'
mem["offset"] is not None
and "identifier" in mem["offset"]
and i_mem["offset"] == "identifier"
)
or (
mem['offset'] is not None
and 'value' in mem['offset']
and i_mem['offset'] == 'imd'
mem["offset"] is not None
and "value" in mem["offset"]
and i_mem["offset"] == "imd"
)
)
# check index
and (
mem['index'] == i_mem['index']
or i_mem['index'] == self.WILDCARD
mem["index"] == i_mem["index"]
or i_mem["index"] == self.WILDCARD
or (
mem['index'] is not None
and 'prefix' in mem['index']
and mem['index']['prefix'] == i_mem['index']
mem["index"] is not None
and "prefix" in mem["index"]
and mem["index"]["prefix"] == i_mem["index"]
)
)
# check scale
and (
mem['scale'] == i_mem['scale']
or i_mem['scale'] == self.WILDCARD
or (mem['scale'] != 1 and i_mem['scale'] != 1)
mem["scale"] == i_mem["scale"]
or i_mem["scale"] == self.WILDCARD
or (mem["scale"] != 1 and i_mem["scale"] != 1)
)
# check pre-indexing
and (
i_mem['pre-indexed'] == self.WILDCARD
or ('pre_indexed' in mem) == (i_mem['pre-indexed'])
i_mem["pre-indexed"] == self.WILDCARD
or ("pre_indexed" in mem) == (i_mem["pre-indexed"])
)
# check post-indexing
and (
i_mem['post-indexed'] == self.WILDCARD
or ('post_indexed' in mem) == (i_mem['post-indexed'])
i_mem["post-indexed"] == self.WILDCARD
or ("post_indexed" in mem) == (i_mem["post-indexed"])
)
):
return True
@@ -690,48 +693,48 @@ class MachineModel(object):
if (
# check base
(
(mem['base'] is None and i_mem['base'] is None)
or i_mem['base'] == self.WILDCARD
or self._is_x86_reg_type(i_mem['base'], mem['base'])
(mem["base"] is None and i_mem["base"] is None)
or i_mem["base"] == self.WILDCARD
or self._is_x86_reg_type(i_mem["base"], mem["base"])
)
# check offset
and (
mem['offset'] == i_mem['offset']
or i_mem['offset'] == self.WILDCARD
mem["offset"] == i_mem["offset"]
or i_mem["offset"] == self.WILDCARD
or (
mem['offset'] is not None
and 'identifier' in mem['offset']
and i_mem['offset'] == 'identifier'
mem["offset"] is not None
and "identifier" in mem["offset"]
and i_mem["offset"] == "identifier"
)
or (
mem['offset'] is not None
and 'value' in mem['offset']
mem["offset"] is not None
and "value" in mem["offset"]
and (
i_mem['offset'] == 'imd'
or (i_mem['offset'] is None and mem['offset']['value'] == '0')
i_mem["offset"] == "imd"
or (i_mem["offset"] is None and mem["offset"]["value"] == "0")
)
)
or (
mem['offset'] is not None
and 'identifier' in mem['offset']
and i_mem['offset'] == 'id'
mem["offset"] is not None
and "identifier" in mem["offset"]
and i_mem["offset"] == "id"
)
)
# check index
and (
mem['index'] == i_mem['index']
or i_mem['index'] == self.WILDCARD
mem["index"] == i_mem["index"]
or i_mem["index"] == self.WILDCARD
or (
mem['index'] is not None
and 'name' in mem['index']
and self._is_x86_reg_type(i_mem['index'], mem['index'])
mem["index"] is not None
and "name" in mem["index"]
and self._is_x86_reg_type(i_mem["index"], mem["index"])
)
)
# check scale
and (
mem['scale'] == i_mem['scale']
or i_mem['scale'] == self.WILDCARD
or (mem['scale'] != 1 and i_mem['scale'] != 1)
mem["scale"] == i_mem["scale"]
or i_mem["scale"] == self.WILDCARD
or (mem["scale"] != 1 and i_mem["scale"] != 1)
)
):
return True
@@ -748,4 +751,4 @@ class MachineModel(object):
def __represent_none(self, yaml_obj, data):
"""YAML representation for `None`"""
return yaml_obj.represent_scalar(u'tag:yaml.org,2002:null', u'~')
return yaml_obj.represent_scalar(u"tag:yaml.org,2002:null", u"~")

View File

@@ -12,25 +12,25 @@ class INSTR_FLAGS:
Flags used for unknown or special instructions
"""
LD = 'is_load_instruction'
TP_UNKWN = 'tp_unknown'
LT_UNKWN = 'lt_unknown'
NOT_BOUND = 'not_bound'
HIDDEN_LD = 'hidden_load'
HAS_LD = 'performs_load'
HAS_ST = 'performs_store'
LD = "is_load_instruction"
TP_UNKWN = "tp_unknown"
LT_UNKWN = "lt_unknown"
NOT_BOUND = "not_bound"
HIDDEN_LD = "hidden_load"
HAS_LD = "performs_load"
HAS_ST = "performs_store"
class ISASemantics(object):
GAS_SUFFIXES = 'bswlqt'
GAS_SUFFIXES = "bswlqt"
def __init__(self, isa, path_to_yaml=None):
self._isa = isa.lower()
path = path_to_yaml or utils.find_datafile('isa/' + self._isa + '.yml')
path = path_to_yaml or utils.find_datafile("isa/" + self._isa + ".yml")
self._isa_model = MachineModel(path_to_yaml=path)
if self._isa == 'x86':
if self._isa == "x86":
self._parser = ParserX86ATT()
elif self._isa == 'aarch64':
elif self._isa == "aarch64":
self._parser = ParserAArch64()
def process(self, instruction_forms):
@@ -45,26 +45,26 @@ class ISASemantics(object):
def assign_src_dst(self, instruction_form):
"""Update instruction form dictionary with source, destination and flag information."""
# if the instruction form doesn't have operands or is None, there's nothing to do
if instruction_form['operands'] is None or instruction_form['instruction'] is None:
instruction_form['semantic_operands'] = AttrDict(
{'source': [], 'destination': [], 'src_dst': []}
if instruction_form["operands"] is None or instruction_form["instruction"] is None:
instruction_form["semantic_operands"] = AttrDict(
{"source": [], "destination": [], "src_dst": []}
)
return
# check if instruction form is in ISA yaml, otherwise apply standard operand assignment
# (one dest, others source)
isa_data = self._isa_model.get_instruction(
instruction_form['instruction'], instruction_form['operands']
instruction_form["instruction"], instruction_form["operands"]
)
if (
isa_data is None
and self._isa == 'x86'
and instruction_form['instruction'][-1] in self.GAS_SUFFIXES
and self._isa == "x86"
and instruction_form["instruction"][-1] in self.GAS_SUFFIXES
):
# Check for instruction without GAS suffix
isa_data = self._isa_model.get_instruction(
instruction_form['instruction'][:-1], instruction_form['operands']
instruction_form["instruction"][:-1], instruction_form["operands"]
)
operands = instruction_form['operands']
operands = instruction_form["operands"]
op_dict = {}
assign_default = False
if isa_data:
@@ -74,52 +74,52 @@ class ISASemantics(object):
# Couldn't found instruction form in ISA DB
assign_default = True
# check for equivalent register-operands DB entry if LD/ST
if any(['memory' in op for op in operands]):
operands_reg = self.substitute_mem_address(instruction_form['operands'])
if any(["memory" in op for op in operands]):
operands_reg = self.substitute_mem_address(instruction_form["operands"])
isa_data_reg = self._isa_model.get_instruction(
instruction_form['instruction'], operands_reg
instruction_form["instruction"], operands_reg
)
if (
isa_data_reg is None
and self._isa == 'x86'
and instruction_form['instruction'][-1] in self.GAS_SUFFIXES
and self._isa == "x86"
and instruction_form["instruction"][-1] in self.GAS_SUFFIXES
):
# Check for instruction without GAS suffix
isa_data_reg = self._isa_model.get_instruction(
instruction_form['instruction'][:-1], operands_reg
instruction_form["instruction"][:-1], operands_reg
)
if isa_data_reg:
assign_default = False
op_dict = self._apply_found_ISA_data(isa_data_reg, operands)
if assign_default:
# no irregular operand structure, apply default
op_dict['source'] = self._get_regular_source_operands(instruction_form)
op_dict['destination'] = self._get_regular_destination_operands(instruction_form)
op_dict['src_dst'] = []
op_dict["source"] = self._get_regular_source_operands(instruction_form)
op_dict["destination"] = self._get_regular_destination_operands(instruction_form)
op_dict["src_dst"] = []
# post-process pre- and post-indexing for aarch64 memory operands
if self._isa == 'aarch64':
for operand in [op for op in op_dict['source'] if 'memory' in op]:
if ('post_indexed' in operand['memory'] and operand['memory']['post_indexed']) or (
'pre_indexed' in operand['memory'] and operand['memory']['pre_indexed']
if self._isa == "aarch64":
for operand in [op for op in op_dict["source"] if "memory" in op]:
if ("post_indexed" in operand["memory"] and operand["memory"]["post_indexed"]) or (
"pre_indexed" in operand["memory"] and operand["memory"]["pre_indexed"]
):
op_dict['src_dst'].append(AttrDict.convert_dict(
{'register': operand['memory']['base']}))
for operand in [op for op in op_dict['destination'] if 'memory' in op]:
if ('post_indexed' in operand['memory'] and operand['memory']['post_indexed']) or (
'pre_indexed' in operand['memory'] and operand['memory']['pre_indexed']
):
op_dict['src_dst'].append(AttrDict.convert_dict(
{'register': operand['memory']['base']}))
# store operand list in dict and reassign operand key/value pair
instruction_form['semantic_operands'] = AttrDict.convert_dict(op_dict)
# assign LD/ST flags
instruction_form['flags'] = (
instruction_form['flags'] if 'flags' in instruction_form else []
op_dict["src_dst"].append(
AttrDict.convert_dict({"register": operand["memory"]["base"]})
)
for operand in [op for op in op_dict["destination"] if "memory" in op]:
if ("post_indexed" in operand["memory"] and operand["memory"]["post_indexed"]) or (
"pre_indexed" in operand["memory"] and operand["memory"]["pre_indexed"]
):
op_dict["src_dst"].append(
AttrDict.convert_dict({"register": operand["memory"]["base"]})
)
# store operand list in dict and reassign operand key/value pair
instruction_form["semantic_operands"] = AttrDict.convert_dict(op_dict)
# assign LD/ST flags
instruction_form["flags"] = instruction_form["flags"] if "flags" in instruction_form else []
if self._has_load(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_LD]
instruction_form["flags"] += [INSTR_FLAGS.HAS_LD]
if self._has_store(instruction_form):
instruction_form['flags'] += [INSTR_FLAGS.HAS_ST]
instruction_form["flags"] += [INSTR_FLAGS.HAS_ST]
def _apply_found_ISA_data(self, isa_data, operands):
"""
@@ -131,34 +131,34 @@ class ISASemantics(object):
:returns: `dict` -- operands dictionary with src/dst assignment
"""
op_dict = {}
op_dict['source'] = []
op_dict['destination'] = []
op_dict['src_dst'] = []
for i, op in enumerate(isa_data['operands']):
if op['source'] and op['destination']:
op_dict['src_dst'].append(operands[i])
op_dict["source"] = []
op_dict["destination"] = []
op_dict["src_dst"] = []
for i, op in enumerate(isa_data["operands"]):
if op["source"] and op["destination"]:
op_dict["src_dst"].append(operands[i])
continue
if op['source']:
op_dict['source'].append(operands[i])
if op["source"]:
op_dict["source"].append(operands[i])
continue
if op['destination']:
op_dict['destination'].append(operands[i])
if op["destination"]:
op_dict["destination"].append(operands[i])
continue
# check for hidden operands like flags or registers
if 'hidden_operands' in isa_data:
if "hidden_operands" in isa_data:
# add operand(s) to semantic_operands of instruction form
for op in isa_data['hidden_operands']:
for op in isa_data["hidden_operands"]:
dict_key = (
'src_dst'
if op['source'] and op['destination']
else 'source'
if op['source']
else 'destination'
"src_dst"
if op["source"] and op["destination"]
else "source"
if op["source"]
else "destination"
)
hidden_op = {op['class']: {}}
key_filter = ['class', 'source', 'destination']
hidden_op = {op["class"]: {}}
key_filter = ["class", "source", "destination"]
for key in [k for k in op.keys() if k not in key_filter]:
hidden_op[op['class']][key] = op[key]
hidden_op[op["class"]][key] = op[key]
hidden_op = AttrDict.convert_dict(hidden_op)
op_dict[dict_key].append(hidden_op)
return op_dict
@@ -166,54 +166,54 @@ class ISASemantics(object):
def _has_load(self, instruction_form):
"""Check if instruction form performs a LOAD"""
for operand in chain(
instruction_form['semantic_operands']['source'],
instruction_form['semantic_operands']['src_dst'],
instruction_form["semantic_operands"]["source"],
instruction_form["semantic_operands"]["src_dst"],
):
if 'memory' in operand:
if "memory" in operand:
return True
return False
def _has_store(self, instruction_form):
"""Check if instruction form perfroms a STORE"""
for operand in chain(
instruction_form['semantic_operands']['destination'],
instruction_form['semantic_operands']['src_dst'],
instruction_form["semantic_operands"]["destination"],
instruction_form["semantic_operands"]["src_dst"],
):
if 'memory' in operand:
if "memory" in operand:
return True
return False
def _get_regular_source_operands(self, instruction_form):
"""Get source operand of given instruction form assuming regular src/dst behavior."""
# if there is only one operand, assume it is a source operand
if len(instruction_form['operands']) == 1:
return [instruction_form['operands'][0]]
if self._isa == 'x86':
if len(instruction_form["operands"]) == 1:
return [instruction_form["operands"][0]]
if self._isa == "x86":
# return all but last operand
return [op for op in instruction_form['operands'][0:-1]]
elif self._isa == 'aarch64':
return [op for op in instruction_form['operands'][1:]]
return [op for op in instruction_form["operands"][0:-1]]
elif self._isa == "aarch64":
return [op for op in instruction_form["operands"][1:]]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))
def _get_regular_destination_operands(self, instruction_form):
"""Get destination operand of given instruction form assuming regular src/dst behavior."""
# if there is only one operand, assume no destination
if len(instruction_form['operands']) == 1:
if len(instruction_form["operands"]) == 1:
return []
if self._isa == 'x86':
if self._isa == "x86":
# return last operand
return instruction_form['operands'][-1:]
if self._isa == 'aarch64':
return instruction_form["operands"][-1:]
if self._isa == "aarch64":
# return first operand
return instruction_form['operands'][:1]
return instruction_form["operands"][:1]
else:
raise ValueError("Unsupported ISA {}.".format(self._isa))
def substitute_mem_address(self, operands):
"""Create memory wildcard for all memory operands"""
return [self._create_reg_wildcard() if 'memory' in op else op for op in operands]
return [self._create_reg_wildcard() if "memory" in op else op for op in operands]
def _create_reg_wildcard(self):
"""Wildcard constructor"""
return {'*': '*'}
return {"*": "*"}

View File

@@ -30,34 +30,34 @@ class KernelDG(nx.DiGraph):
# 3. get LT value and set as edge weight
dg = nx.DiGraph()
for i, instruction_form in enumerate(kernel):
dg.add_node(instruction_form['line_number'])
dg.nodes[instruction_form['line_number']]['instruction_form'] = instruction_form
dg.add_node(instruction_form["line_number"])
dg.nodes[instruction_form["line_number"]]["instruction_form"] = instruction_form
# add load as separate node if existent
if (
INSTR_FLAGS.HAS_LD in instruction_form['flags']
and INSTR_FLAGS.LD not in instruction_form['flags']
INSTR_FLAGS.HAS_LD in instruction_form["flags"]
and INSTR_FLAGS.LD not in instruction_form["flags"]
):
# add new node
dg.add_node(instruction_form['line_number'] + 0.1)
dg.nodes[instruction_form['line_number'] + 0.1][
'instruction_form'
dg.add_node(instruction_form["line_number"] + 0.1)
dg.nodes[instruction_form["line_number"] + 0.1][
"instruction_form"
] = instruction_form
# and set LD latency as edge weight
dg.add_edge(
instruction_form['line_number'] + 0.1,
instruction_form['line_number'],
latency=instruction_form['latency'] - instruction_form['latency_wo_load'],
instruction_form["line_number"] + 0.1,
instruction_form["line_number"],
latency=instruction_form["latency"] - instruction_form["latency_wo_load"],
)
for dep in self.find_depending(instruction_form, kernel[i + 1 :]):
edge_weight = (
instruction_form['latency']
if 'latency_wo_load' not in instruction_form
else instruction_form['latency_wo_load']
instruction_form["latency"]
if "latency_wo_load" not in instruction_form
else instruction_form["latency_wo_load"]
)
dg.add_edge(
instruction_form['line_number'], dep['line_number'], latency=edge_weight
instruction_form["line_number"], dep["line_number"], latency=edge_weight
)
dg.nodes[dep['line_number']]['instruction_form'] = dep
dg.nodes[dep["line_number"]]["instruction_form"] = dep
return dg
def check_for_loopcarried_dep(self, kernel):
@@ -109,17 +109,17 @@ class KernelDG(nx.DiGraph):
for dep in loopcarried_deps:
nodes = []
for n in dep[1]:
self._get_node_by_lineno(int(n))['latency_lcd'] = 0
self._get_node_by_lineno(int(n))["latency_lcd"] = 0
for n in dep[1]:
node = self._get_node_by_lineno(int(n))
if int(n) != n and int(n) in dep[1]:
node['latency_lcd'] += node['latency'] - node['latency_wo_load']
node["latency_lcd"] += node["latency"] - node["latency_wo_load"]
else:
node['latency_lcd'] += node['latency_wo_load']
node["latency_lcd"] += node["latency_wo_load"]
nodes.append(node)
loopcarried_deps_dict[dep[0]] = {
'root': self._get_node_by_lineno(dep[0]),
'dependencies': nodes,
"root": self._get_node_by_lineno(dep[0]),
"dependencies": nodes,
}
return loopcarried_deps_dict
@@ -131,30 +131,30 @@ class KernelDG(nx.DiGraph):
def get_critical_path(self):
"""Find and return critical path after the creation of a directed graph."""
if nx.algorithms.dag.is_directed_acyclic_graph(self.dg):
longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight='latency')
longest_path = nx.algorithms.dag.dag_longest_path(self.dg, weight="latency")
for line_number in longest_path:
self._get_node_by_lineno(int(line_number))['latency_cp'] = 0
self._get_node_by_lineno(int(line_number))["latency_cp"] = 0
# add LD latency to instruction
for line_number in longest_path:
node = self._get_node_by_lineno(int(line_number))
if line_number != int(line_number) and int(line_number) in longest_path:
node['latency_cp'] += self.dg.edges[(line_number, int(line_number))]['latency']
node["latency_cp"] += self.dg.edges[(line_number, int(line_number))]["latency"]
elif (
line_number == int(line_number)
and 'mem_dep' in node
and self.dg.has_edge(node['mem_dep']['line_number'], line_number)
and "mem_dep" in node
and self.dg.has_edge(node["mem_dep"]["line_number"], line_number)
):
node['latency_cp'] += node['latency']
node["latency_cp"] += node["latency"]
else:
node['latency_cp'] += (
node['latency']
if 'latency_wo_load' not in node
else node['latency_wo_load']
node["latency_cp"] += (
node["latency"]
if "latency_wo_load" not in node
else node["latency_wo_load"]
)
return [x for x in self.kernel if x['line_number'] in longest_path]
return [x for x in self.kernel if x["line_number"] in longest_path]
else:
# split to DAG
raise NotImplementedError('Kernel is cyclic.')
raise NotImplementedError("Kernel is cyclic.")
def get_loopcarried_dependencies(self):
"""
@@ -164,7 +164,7 @@ class KernelDG(nx.DiGraph):
return self.loopcarried_deps
else:
# split to DAG
raise NotImplementedError('Kernel is cyclic.')
raise NotImplementedError("Kernel is cyclic.")
def find_depending(
self, instruction_form, kernel, include_write=False, flag_dependencies=False
@@ -186,7 +186,7 @@ class KernelDG(nx.DiGraph):
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'register' in dst:
if "register" in dst:
# Check for read of register until overwrite
for instr_form in kernel:
if self.is_read(dst.register, instr_form):
@@ -200,7 +200,7 @@ class KernelDG(nx.DiGraph):
if include_write:
yield instr_form
break
if 'flag' in dst and flag_dependencies:
if "flag" in dst and flag_dependencies:
# Check for read of flag until overwrite
for instr_form in kernel:
if self.is_read(dst.flag, instr_form):
@@ -214,23 +214,23 @@ class KernelDG(nx.DiGraph):
if include_write:
yield instr_form
break
elif 'memory' in dst:
elif "memory" in dst:
# Check if base register is altered during memory access
if 'pre_indexed' in dst.memory or 'post_indexed' in dst.memory:
if "pre_indexed" in dst.memory or "post_indexed" in dst.memory:
# Check for read of base register until overwrite
for instr_form in kernel:
if self.is_read(dst.memory.base, instr_form):
instr_form['mem_dep'] = instruction_form
instr_form["mem_dep"] = instruction_form
yield instr_form
if self.is_written(dst.memory.base, instr_form):
# operand in src_dst list
if include_write:
instr_form['mem_dep'] = instruction_form
instr_form["mem_dep"] = instruction_form
yield instr_form
break
elif self.is_written(dst.memory.base, instr_form):
if include_write:
instr_form['mem_dep'] = instruction_form
instr_form["mem_dep"] = instruction_form
yield instr_form
break
@@ -239,8 +239,8 @@ class KernelDG(nx.DiGraph):
Returns iterator
"""
if not instr_form and not line_number:
raise ValueError('Either instruction form or line_number required.')
line_number = line_number if line_number else instr_form['line_number']
raise ValueError("Either instruction form or line_number required.")
line_number = line_number if line_number else instr_form["line_number"]
if self.dg.has_node(line_number):
return self.dg.successors(line_number)
return iter([])
@@ -253,29 +253,25 @@ class KernelDG(nx.DiGraph):
for src in chain(
instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst
):
if 'register' in src:
if "register" in src:
is_read = self.parser.is_reg_dependend_of(register, src.register) or is_read
if 'flag' in src:
if "flag" in src:
is_read = self.parser.is_flag_dependend_of(register, src.flag) or is_read
if 'memory' in src:
if "memory" in src:
if src.memory.base is not None:
is_read = self.parser.is_reg_dependend_of(register, src.memory.base) or is_read
if src.memory.index is not None:
is_read = (
self.parser.is_reg_dependend_of(register, src.memory.index) or is_read
)
is_read = self.parser.is_reg_dependend_of(register, src.memory.index) or is_read
# Check also if read in destination memory address
for dst in chain(
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'memory' in dst:
if "memory" in dst:
if dst.memory.base is not None:
is_read = self.parser.is_reg_dependend_of(register, dst.memory.base) or is_read
if dst.memory.index is not None:
is_read = (
self.parser.is_reg_dependend_of(register, dst.memory.index) or is_read
)
is_read = self.parser.is_reg_dependend_of(register, dst.memory.index) or is_read
return is_read
def is_written(self, register, instruction_form):
@@ -287,12 +283,12 @@ class KernelDG(nx.DiGraph):
instruction_form.semantic_operands.destination,
instruction_form.semantic_operands.src_dst,
):
if 'register' in dst:
if "register" in dst:
is_written = self.parser.is_reg_dependend_of(register, dst.register) or is_written
if 'flag' in dst:
if "flag" in dst:
is_written = self.parser.is_flag_dependend_of(register, dst.flag) or is_written
if 'memory' in dst:
if 'pre_indexed' in dst.memory or 'post_indexed' in dst.memory:
if "memory" in dst:
if "pre_indexed" in dst.memory or "post_indexed" in dst.memory:
is_written = (
self.parser.is_reg_dependend_of(register, dst.memory.base) or is_written
)
@@ -300,8 +296,8 @@ class KernelDG(nx.DiGraph):
for src in chain(
instruction_form.semantic_operands.source, instruction_form.semantic_operands.src_dst
):
if 'memory' in src:
if 'pre_indexed' in src.memory or 'post_indexed' in src.memory:
if "memory" in src:
if "pre_indexed" in src.memory or "post_indexed" in src.memory:
is_written = (
self.parser.is_reg_dependend_of(register, src.memory.base) or is_written
)
@@ -317,51 +313,51 @@ class KernelDG(nx.DiGraph):
"""
graph = copy.deepcopy(self.dg)
cp = self.get_critical_path()
cp_line_numbers = [x['line_number'] for x in cp]
cp_line_numbers = [x["line_number"] for x in cp]
lcd = self.get_loopcarried_dependencies()
lcd_line_numbers = {}
for dep in lcd:
lcd_line_numbers[dep] = [x['line_number'] for x in lcd[dep]['dependencies']]
lcd_line_numbers[dep] = [x["line_number"] for x in lcd[dep]["dependencies"]]
# add color scheme
graph.graph['node'] = {'colorscheme': 'accent8'}
graph.graph['edge'] = {'colorscheme': 'accent8'}
graph.graph["node"] = {"colorscheme": "accent8"}
graph.graph["edge"] = {"colorscheme": "accent8"}
# create LCD edges
for dep in lcd_line_numbers:
min_line_number = min(lcd_line_numbers[dep])
max_line_number = max(lcd_line_numbers[dep])
graph.add_edge(max_line_number, min_line_number)
graph.edges[max_line_number, min_line_number]['latency'] = [
x for x in lcd[dep]['dependencies'] if x['line_number'] == max_line_number
][0]['latency_lcd']
graph.edges[max_line_number, min_line_number]["latency"] = [
x for x in lcd[dep]["dependencies"] if x["line_number"] == max_line_number
][0]["latency_lcd"]
# add label to edges
for e in graph.edges:
graph.edges[e]['label'] = graph.edges[e]['latency']
graph.edges[e]["label"] = graph.edges[e]["latency"]
# add CP values to graph
for n in cp:
graph.nodes[n['line_number']]['instruction_form']['latency_cp'] = n['latency_cp']
graph.nodes[n["line_number"]]["instruction_form"]["latency_cp"] = n["latency_cp"]
# color CP and LCD
for n in graph.nodes:
if n in cp_line_numbers:
# graph.nodes[n]['color'] = 1
graph.nodes[n]['style'] = 'bold'
graph.nodes[n]['penwidth'] = 4
graph.nodes[n]["style"] = "bold"
graph.nodes[n]["penwidth"] = 4
for col, dep in enumerate(lcd):
if n in lcd_line_numbers[dep]:
if 'style' not in graph.nodes[n]:
graph.nodes[n]['style'] = 'filled'
if "style" not in graph.nodes[n]:
graph.nodes[n]["style"] = "filled"
else:
graph.nodes[n]['style'] += ',filled'
graph.nodes[n]['fillcolor'] = 2 + col
graph.nodes[n]["style"] += ",filled"
graph.nodes[n]["fillcolor"] = 2 + col
# color edges
for e in graph.edges:
if (
graph.nodes[e[0]]['instruction_form']['line_number'] in cp_line_numbers
and graph.nodes[e[1]]['instruction_form']['line_number'] in cp_line_numbers
graph.nodes[e[0]]["instruction_form"]["line_number"] in cp_line_numbers
and graph.nodes[e[1]]["instruction_form"]["line_number"] in cp_line_numbers
and e[0] < e[1]
):
bold_edge = True
@@ -369,38 +365,38 @@ class KernelDG(nx.DiGraph):
if i in cp_line_numbers:
bold_edge = False
if bold_edge:
graph.edges[e]['style'] = 'bold'
graph.edges[e]['penwidth'] = 3
graph.edges[e]["style"] = "bold"
graph.edges[e]["penwidth"] = 3
for dep in lcd_line_numbers:
if (
graph.nodes[e[0]]['instruction_form']['line_number'] in lcd_line_numbers[dep]
and graph.nodes[e[1]]['instruction_form']['line_number']
graph.nodes[e[0]]["instruction_form"]["line_number"] in lcd_line_numbers[dep]
and graph.nodes[e[1]]["instruction_form"]["line_number"]
in lcd_line_numbers[dep]
):
graph.edges[e]['color'] = graph.nodes[e[1]]['fillcolor']
graph.edges[e]["color"] = graph.nodes[e[1]]["fillcolor"]
# rename node from [idx] to [idx mnemonic] and add shape
mapping = {}
for n in graph.nodes:
if int(n) != n:
mapping[n] = '{}: LOAD'.format(int(n))
graph.nodes[n]['fontname'] = 'italic'
graph.nodes[n]['fontsize'] = 11.0
mapping[n] = "{}: LOAD".format(int(n))
graph.nodes[n]["fontname"] = "italic"
graph.nodes[n]["fontsize"] = 11.0
else:
node = graph.nodes[n]['instruction_form']
if node['instruction'] is not None:
mapping[n] = '{}: {}'.format(n, node['instruction'])
node = graph.nodes[n]["instruction_form"]
if node["instruction"] is not None:
mapping[n] = "{}: {}".format(n, node["instruction"])
else:
label = 'label' if node['label'] else None
label = 'directive' if node['directive'] else label
label = 'comment' if node['comment'] and label is None else label
mapping[n] = '{}: {}'.format(n, label)
graph.nodes[n]['fontname'] = 'italic'
graph.nodes[n]['fontsize'] = 11.0
graph.nodes[n]['shape'] = 'rectangle'
label = "label" if node["label"] else None
label = "directive" if node["directive"] else label
label = "comment" if node["comment"] and label is None else label
mapping[n] = "{}: {}".format(n, label)
graph.nodes[n]["fontname"] = "italic"
graph.nodes[n]["fontsize"] = 11.0
graph.nodes[n]["shape"] = "rectangle"
nx.relabel.relabel_nodes(graph, mapping, copy=False)
if filepath:
nx.drawing.nx_agraph.write_dot(graph, filepath)
else:
nx.drawing.nx_agraph.write_dot(graph, 'osaca_dg.dot')
nx.drawing.nx_agraph.write_dot(graph, "osaca_dg.dot")

View File

@@ -3,7 +3,7 @@ from collections import OrderedDict
from osaca.parser import ParserAArch64, ParserX86ATT, get_parser
COMMENT_MARKER = {'start': 'OSACA-BEGIN', 'end': 'OSACA-END'}
COMMENT_MARKER = {"start": "OSACA-BEGIN", "end": "OSACA-END"}
def reduce_to_section(kernel, isa):
@@ -15,12 +15,12 @@ def reduce_to_section(kernel, isa):
:returns: `list` -- marked section of kernel as list of instruction forms
"""
isa = isa.lower()
if isa == 'x86':
if isa == "x86":
start, end = find_marked_kernel_x86ATT(kernel)
elif isa == 'aarch64':
elif isa == "aarch64":
start, end = find_marked_kernel_AArch64(kernel)
else:
raise ValueError('ISA not supported.')
raise ValueError("ISA not supported.")
if start == -1:
start = 0
if end == -1:
@@ -35,12 +35,12 @@ def find_marked_kernel_AArch64(lines):
:param list lines: kernel
:returns: `tuple of int` -- start and end line of marked section
"""
nop_bytes = ['213', '3', '32', '31']
nop_bytes = ["213", "3", "32", "31"]
return find_marked_section(
lines,
ParserAArch64(),
['mov'],
'x1',
["mov"],
"x1",
[111, 222],
nop_bytes,
reverse=True,
@@ -55,12 +55,12 @@ def find_marked_kernel_x86ATT(lines):
:param list lines: kernel
:returns: `tuple of int` -- start and end line of marked section
"""
nop_bytes = ['100', '103', '144']
nop_bytes = ["100", "103", "144"]
return find_marked_section(
lines,
ParserX86ATT(),
['mov', 'movl'],
'ebx',
["mov", "movl"],
"ebx",
[111, 222],
nop_bytes,
comments=COMMENT_MARKER,
@@ -70,32 +70,32 @@ def find_marked_kernel_x86ATT(lines):
def get_marker(isa, comment=""):
"""Return tuple of start and end marker lines."""
isa = isa.lower()
if isa == 'x86':
if isa == "x86":
start_marker_raw = (
'movl $111, %ebx # OSACA START MARKER\n'
'.byte 100 # OSACA START MARKER\n'
'.byte 103 # OSACA START MARKER\n'
'.byte 144 # OSACA START MARKER\n'
"movl $111, %ebx # OSACA START MARKER\n"
".byte 100 # OSACA START MARKER\n"
".byte 103 # OSACA START MARKER\n"
".byte 144 # OSACA START MARKER\n"
)
if comment:
start_marker_raw += "# {}\n".format(comment)
end_marker_raw = (
'movl $222, %ebx # OSACA END MARKER\n'
'.byte 100 # OSACA END MARKER\n'
'.byte 103 # OSACA END MARKER\n'
'.byte 144 # OSACA END MARKER\n'
"movl $222, %ebx # OSACA END MARKER\n"
".byte 100 # OSACA END MARKER\n"
".byte 103 # OSACA END MARKER\n"
".byte 144 # OSACA END MARKER\n"
)
elif isa == 'aarch64':
elif isa == "aarch64":
start_marker_raw = (
'mov x1, #111 // OSACA START MARKER\n'
'.byte 213,3,32,31 // OSACA START MARKER\n'
"mov x1, #111 // OSACA START MARKER\n"
".byte 213,3,32,31 // OSACA START MARKER\n"
)
if comment:
start_marker_raw += "// {}\n".format(comment)
# After loop
end_marker_raw = (
'mov x1, #222 // OSACA END MARKER\n'
'.byte 213,3,32,31 // OSACA END MARKER\n'
"mov x1, #222 // OSACA END MARKER\n"
".byte 213,3,32,31 // OSACA END MARKER\n"
)
parser = get_parser(isa)
@@ -134,18 +134,18 @@ def find_marked_section(
for i, line in enumerate(lines):
try:
if line.instruction is None and comments is not None and line.comment is not None:
if comments['start'] == line.comment:
if comments["start"] == line.comment:
index_start = i + 1
elif comments['end'] == line.comment:
elif comments["end"] == line.comment:
index_end = i
elif line.instruction in mov_instr and lines[i + 1].directive is not None:
source = line.operands[0 if not reverse else 1]
destination = line.operands[1 if not reverse else 0]
# instruction pair matches, check for operands
if (
'immediate' in source
"immediate" in source
and parser.normalize_imd(source.immediate) == mov_vals[0]
and 'register' in destination
and "register" in destination
and parser.get_full_reg_name(destination.register) == mov_reg
):
# operands of first instruction match start, check for second one
@@ -154,9 +154,9 @@ def find_marked_section(
# return first line after the marker
index_start = i + 1 + line_count
elif (
'immediate' in source
"immediate" in source
and parser.normalize_imd(source.immediate) == mov_vals[1]
and 'register' in destination
and "register" in destination
and parser.get_full_reg_name(destination.register) == mov_reg
):
# operand of first instruction match end, check for second one
@@ -179,7 +179,7 @@ def match_bytes(lines, index, byte_list):
while (
index < len(lines)
and lines[index].directive is not None
and lines[index].directive.name == 'byte'
and lines[index].directive.name == "byte"
):
line_count += 1
extracted_bytes += lines[index].directive.parameters
@@ -199,14 +199,14 @@ def find_jump_labels(lines):
labels = OrderedDict()
current_label = None
for i, line in enumerate(lines):
if line['label'] is not None:
if line["label"] is not None:
# When a new label is found, add to blocks dict
labels[line['label']] = (i,)
labels[line["label"]] = (i,)
# End previous block at previous line
if current_label is not None:
labels[current_label] = (labels[current_label][0], i)
# Update current block name
current_label = line['label']
current_label = line["label"]
elif current_label is None:
# If no block has been started, skip end detection
continue
@@ -218,9 +218,9 @@ def find_jump_labels(lines):
for label in list(labels):
if all(
[
l['instruction'].startswith('.')
l["instruction"].startswith(".")
for l in lines[labels[label][0] : labels[label][1]]
if l['instruction'] is not None
if l["instruction"] is not None
]
):
del labels[label]
@@ -247,11 +247,11 @@ def find_basic_blocks(lines):
terminate = False
blocks[label].append(line)
# Find end of block by searching for references to valid jump labels
if line['instruction'] and line['operands']:
for operand in [o for o in line['operands'] if 'identifier' in o]:
if operand['identifier']['name'] in valid_jump_labels:
if line["instruction"] and line["operands"]:
for operand in [o for o in line["operands"] if "identifier" in o]:
if operand["identifier"]["name"] in valid_jump_labels:
terminate = True
elif line['label'] is not None:
elif line["label"] is not None:
terminate = True
if terminate:
break
@@ -276,15 +276,15 @@ def find_basic_loop_bodies(lines):
terminate = False
current_block.append(line)
# Find end of block by searching for references to valid jump labels
if line['instruction'] and line['operands']:
if line["instruction"] and line["operands"]:
# Ignore `b.none` instructions (relevant von ARM SVE code)
# This branch instruction is often present _within_ inner loop blocks, but usually
# do not terminate
if line['instruction'] == 'b.none':
if line["instruction"] == "b.none":
continue
for operand in [o for o in line['operands'] if 'identifier' in o]:
if operand['identifier']['name'] in valid_jump_labels:
if operand['identifier']['name'] == label:
for operand in [o for o in line["operands"] if "identifier" in o]:
if operand["identifier"]["name"] in valid_jump_labels:
if operand["identifier"]["name"] == label:
loop_bodies[label] = current_block
terminate = True
break

View File

@@ -1,8 +1,8 @@
#!/usr/bin/env python3
import os.path
DATA_DIRS = [os.path.expanduser('~/.osaca/data'), os.path.join(os.path.dirname(__file__), 'data')]
CACHE_DIR = os.path.expanduser('~/.osaca/cache')
DATA_DIRS = [os.path.expanduser("~/.osaca/data"), os.path.join(os.path.dirname(__file__), "data")]
CACHE_DIR = os.path.expanduser("~/.osaca/cache")
def find_datafile(name):

View File

@@ -4,6 +4,7 @@
from setuptools import setup, find_packages
from setuptools.command.install import install as _install
from setuptools.command.sdist import sdist as _sdist
# To use a consistent encoding
from codecs import open
import os
@@ -16,8 +17,9 @@ here = os.path.abspath(os.path.dirname(__file__))
# Stolen from pip
def read(*names, **kwargs):
with io.open(os.path.join(os.path.dirname(__file__), *names),
encoding=kwargs.get("encoding", "utf8")) as fp:
with io.open(
os.path.join(os.path.dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")
) as fp:
return fp.read()
@@ -32,11 +34,11 @@ def find_version(*file_paths):
def _run_build_cache(dir):
from subprocess import check_call
# This is run inside the install staging directory (that had no .pyc files)
# We don't want to generate any.
# https://github.com/eliben/pycparser/pull/135
check_call([sys.executable, '-B', '_build_cache.py'],
cwd=os.path.join(dir, 'osaca', 'data'))
check_call([sys.executable, "-B", "_build_cache.py"], cwd=os.path.join(dir, "osaca", "data"))
class install(_install):
@@ -52,77 +54,60 @@ class sdist(_sdist):
# Get the long description from the README file
with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
with open(os.path.join(here, "README.rst"), encoding="utf-8") as f:
long_description = f.read()
setup(
name='osaca',
name="osaca",
# Version should comply with PEP440. For a discussion on single-sourcing
# the version across setup.py and the project code, see
# https://packaging.python.org/en/latest/distributing.html
version=find_version('osaca', '__init__.py'),
description='Open Source Architecture Code Analyzer',
version=find_version("osaca", "__init__.py"),
description="Open Source Architecture Code Analyzer",
long_description=long_description,
long_description_content_type='text/x-rst',
long_description_content_type="text/x-rst",
# The project's main homepage
url='https://github.com/RRZE-HPC/OSACA',
url="https://github.com/RRZE-HPC/OSACA",
# Author details
author='Jan Laukemann',
author_email='jan.laukemann@fau.de',
author="Jan Laukemann",
author_email="jan.laukemann@fau.de",
# Choose your license
license='AGPLv3',
license="AGPLv3",
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 4 - Beta',
"Development Status :: 4 - Beta",
# Indicate who your project is intended for
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'Topic :: Scientific/Engineering',
'Topic :: Software Development',
'Topic :: Utilities',
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"Topic :: Scientific/Engineering",
"Topic :: Software Development",
"Topic :: Utilities",
# Pick your license as you wish (should match "license" above)
'License :: OSI Approved :: GNU Affero General Public License v3',
"License :: OSI Approved :: GNU Affero General Public License v3",
# Specify the Python versions you support here. In particular, ensure
# that you indicate wheter you support Python2, Python 3 or both.
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
],
# What doesd your project relate to?
keywords='hpc performance benchmark analysis architecture',
keywords="hpc performance benchmark analysis architecture",
# You can just specify the packages manually here if your project is
# simple. Or you can use find_packages().
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
packages=find_packages(exclude=["contrib", "docs", "tests"]),
# List run-time dependencies here. These will be installed by pip when
# your project is installed. For an analysis of "install_requires" vs pip's
# requirements files see:
# https://packaging.python.org/en/latest/requirements.html
install_requires=[
'networkx',
'pyparsing>=2.3.1',
'ruamel.yaml>=0.15.71',
],
python_requires='>=3.5',
install_requires=["networkx", "pyparsing>=2.3.1", "ruamel.yaml>=0.15.71",],
python_requires=">=3.5",
# List additional groups of dependencies here (e.g. development
# dependencies). You can install these using the following syntax,
# for example:
@@ -131,27 +116,19 @@ setup(
# 'dev': ['check-manifest'],
# 'test': ['coverage'],
# },
# If there are data files included in your packages that need to be
# installed, specify them here. If using Python 2.6 or less, then these
# have to be included in MANIFEST.in as well.
include_package_data=True,
# Although 'package_data' is the preferred approach, in some case you may
# need to place data files outside of your packages. See:
# http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
# In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
# data_files=[('my_data', ['data/data_file'])],
# To provide executable scripts, use entry points in preference to the
# "scripts" keyword. Entry points provide cross-platform support and allow
# pip to create the appropriate form of executable for the target platform.
entry_points={
'console_scripts': [
'osaca=osaca.osaca:main',
],
},
entry_points={"console_scripts": ["osaca=osaca.osaca:main",],},
# Overwriting install and sdist to enforce cache distribution with package
cmdclass={'install': install, 'sdist': sdist},
cmdclass={"install": install, "sdist": sdist},
)

View File

@@ -3,18 +3,18 @@
import sys
import unittest
sys.path[0:0] = ['.', '..']
sys.path[0:0] = [".", ".."]
suite = unittest.TestLoader().loadTestsFromNames(
[
'test_base_parser',
'test_parser_x86att',
'test_parser_AArch64',
'test_marker_utils',
'test_semantics',
'test_frontend',
'test_db_interface',
'test_kerncraftAPI',
'test_cli',
"test_base_parser",
"test_parser_x86att",
"test_parser_AArch64",
"test_marker_utils",
"test_semantics",
"test_frontend",
"test_db_interface",
"test_kerncraftAPI",
"test_cli",
]
)

View File

@@ -16,13 +16,13 @@ class TestBaseParser(unittest.TestCase):
self.parser = BaseParser()
except NotImplementedError:
pass
with open(self._find_file('triad_x86_iaca.s')) as f:
with open(self._find_file("triad_x86_iaca.s")) as f:
self.triad_code = f.read()
with open(self._find_file('triad_arm_iaca.s')) as f:
with open(self._find_file("triad_arm_iaca.s")) as f:
self.triad_code_arm = f.read()
with open(self._find_file('kernel_x86.s')) as f:
with open(self._find_file("kernel_x86.s")) as f:
self.x86_code = f.read()
with open(self._find_file('kernel_aarch64.s')) as f:
with open(self._find_file("kernel_aarch64.s")) as f:
self.aarch64_code = f.read()
##################
@@ -34,19 +34,19 @@ class TestBaseParser(unittest.TestCase):
self.parser.parse_file(self.triad_code)
def test_parse_line(self):
line_instruction = '\t\tlea 2(%rax,%rax), %ecx #12.9'
line_instruction = "\t\tlea 2(%rax,%rax), %ecx #12.9"
with self.assertRaises(NotImplementedError):
self.parser.parse_line(line_instruction)
def test_parse_instruction(self):
instr1 = '\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27'
instr1 = "\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27"
with self.assertRaises(NotImplementedError):
self.parser.parse_instruction(instr1)
def test_register_funcs(self):
reg_a1 = AttrDict({'name': 'rax'})
reg_a2 = AttrDict({'name': 'eax'})
register_string = 'v1.2d'
reg_a1 = AttrDict({"name": "rax"})
reg_a2 = AttrDict({"name": "eax"})
register_string = "v1.2d"
with self.assertRaises(NotImplementedError):
self.parser.is_reg_dependend_of(reg_a1, reg_a2)
with self.assertRaises(NotImplementedError):
@@ -61,15 +61,15 @@ class TestBaseParser(unittest.TestCase):
self.parser.get_full_reg_name(reg_a1)
def test_normalize_imd(self):
imd_hex_1 = {'value': '0x4f'}
imd_hex_1 = {"value": "0x4f"}
with self.assertRaises(NotImplementedError):
self.parser.normalize_imd(imd_hex_1)
def test_detect_ISA(self):
self.assertEqual(BaseParser.detect_ISA(self.triad_code), 'x86')
self.assertEqual(BaseParser.detect_ISA(self.triad_code_arm), 'aarch64')
self.assertEqual(BaseParser.detect_ISA(self.x86_code), 'x86')
self.assertEqual(BaseParser.detect_ISA(self.aarch64_code), 'aarch64')
self.assertEqual(BaseParser.detect_ISA(self.triad_code), "x86")
self.assertEqual(BaseParser.detect_ISA(self.triad_code_arm), "aarch64")
self.assertEqual(BaseParser.detect_ISA(self.x86_code), "x86")
self.assertEqual(BaseParser.detect_ISA(self.aarch64_code), "aarch64")
##################
# Helper functions
@@ -78,11 +78,11 @@ class TestBaseParser(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestBaseParser)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -29,11 +29,11 @@ class TestCLI(unittest.TestCase):
def test_check_arguments(self):
parser = osaca.create_parser(parser=ErrorRaisingArgumentParser())
args = parser.parse_args(['--arch', 'WRONG_ARCH', self._find_file('gs', 'csx', 'gcc')])
args = parser.parse_args(["--arch", "WRONG_ARCH", self._find_file("gs", "csx", "gcc")])
with self.assertRaises(ValueError):
osaca.check_arguments(args, parser)
args = parser.parse_args(
['--arch', 'csx', '--import', 'WRONG_BENCH', self._find_file('gs', 'csx', 'gcc')]
["--arch", "csx", "--import", "WRONG_BENCH", self._find_file("gs", "csx", "gcc")]
)
with self.assertRaises(ValueError):
osaca.check_arguments(args, parser)
@@ -42,22 +42,22 @@ class TestCLI(unittest.TestCase):
parser = osaca.create_parser(parser=ErrorRaisingArgumentParser())
args = parser.parse_args(
[
'--arch',
'tx2',
'--import',
'ibench',
self._find_test_file('ibench_import_aarch64.dat'),
"--arch",
"tx2",
"--import",
"ibench",
self._find_test_file("ibench_import_aarch64.dat"),
]
)
output = StringIO()
osaca.run(args, output_file=output)
args = parser.parse_args(
[
'--arch',
'tx2',
'--import',
'asmbench',
self._find_test_file('asmbench_import_aarch64.dat'),
"--arch",
"tx2",
"--import",
"asmbench",
self._find_test_file("asmbench_import_aarch64.dat"),
]
)
osaca.run(args, output_file=output)
@@ -65,28 +65,28 @@ class TestCLI(unittest.TestCase):
def test_check_db(self):
parser = osaca.create_parser(parser=ErrorRaisingArgumentParser())
args = parser.parse_args(
['--arch', 'tx2', '--db-check', '--verbose', self._find_test_file('triad_x86_iaca.s')]
["--arch", "tx2", "--db-check", "--verbose", self._find_test_file("triad_x86_iaca.s")]
)
output = StringIO()
osaca.run(args, output_file=output)
def test_get_parser(self):
self.assertTrue(isinstance(osaca.get_asm_parser('csx'), ParserX86ATT))
self.assertTrue(isinstance(osaca.get_asm_parser('tx2'), ParserAArch64))
self.assertTrue(isinstance(osaca.get_asm_parser("csx"), ParserX86ATT))
self.assertTrue(isinstance(osaca.get_asm_parser("tx2"), ParserAArch64))
with self.assertRaises(ValueError):
osaca.get_asm_parser('UNKNOWN')
osaca.get_asm_parser("UNKNOWN")
def test_marker_insert_x86(self):
# copy file to add markers
name = self._find_test_file('kernel_x86.s')
name_copy = name + '.copy.s'
name = self._find_test_file("kernel_x86.s")
name_copy = name + ".copy.s"
copyfile(name, name_copy)
user_input = ['.L10']
user_input = [".L10"]
output = StringIO()
parser = osaca.create_parser()
args = parser.parse_args(['--arch', 'csx', '--insert-marker', name_copy])
with patch('builtins.input', side_effect=user_input):
args = parser.parse_args(["--arch", "csx", "--insert-marker", name_copy])
with patch("builtins.input", side_effect=user_input):
osaca.run(args, output_file=output)
lines_orig = len(open(name).readlines())
@@ -97,14 +97,14 @@ class TestCLI(unittest.TestCase):
def test_marker_insert_aarch64(self):
# copy file to add markers
name = self._find_test_file('kernel_aarch64.s')
name_copy = name + '.copy.s'
name = self._find_test_file("kernel_aarch64.s")
name_copy = name + ".copy.s"
copyfile(name, name_copy)
user_input = ['.LBB0_32', '64']
user_input = [".LBB0_32", "64"]
parser = osaca.create_parser()
args = parser.parse_args(['--arch', 'tx2', '--insert-marker', name_copy])
with patch('builtins.input', side_effect=user_input):
args = parser.parse_args(["--arch", "tx2", "--insert-marker", name_copy])
with patch("builtins.input", side_effect=user_input):
osaca.run(args)
lines_orig = len(open(name).readlines())
@@ -115,18 +115,18 @@ class TestCLI(unittest.TestCase):
def test_examples(self):
kernels = [
'add',
'copy',
'daxpy',
'gs',
'j2d',
'striad',
'sum_reduction',
'triad',
'update',
"add",
"copy",
"daxpy",
"gs",
"j2d",
"striad",
"sum_reduction",
"triad",
"update",
]
archs = ['csx', 'tx2', 'zen1']
comps = {'csx': ['gcc', 'icc'], 'tx2': ['gcc', 'clang'], 'zen1': ['gcc']}
archs = ["csx", "tx2", "zen1"]
comps = {"csx": ["gcc", "icc"], "tx2": ["gcc", "clang"], "zen1": ["gcc"]}
parser = osaca.create_parser()
# Analyze all asm files resulting out of kernels, archs and comps
for k in kernels:
@@ -134,11 +134,11 @@ class TestCLI(unittest.TestCase):
for c in comps[a]:
with self.subTest(kernel=k, arch=a, comp=c):
args = parser.parse_args(
['--arch', a, self._find_file(k, a, c), '--export-graph', '/dev/null']
["--arch", a, self._find_file(k, a, c), "--export-graph", "/dev/null"]
)
output = StringIO()
osaca.run(args, output_file=output)
self.assertTrue('WARNING' not in output.getvalue())
self.assertTrue("WARNING" not in output.getvalue())
def test_architectures(self):
parser = osaca.create_parser()
@@ -147,10 +147,8 @@ class TestCLI(unittest.TestCase):
for arch in archs:
with self.subTest(micro_arch=arch):
isa = MachineModel.get_isa_for_arch(arch)
kernel = 'kernel_{}.s'.format(isa)
args = parser.parse_args(
['--arch', arch, self._find_test_file(kernel)]
)
kernel = "kernel_{}.s".format(isa)
args = parser.parse_args(["--arch", arch, self._find_test_file(kernel)])
output = StringIO()
osaca.run(args, output_file=output)
@@ -168,59 +166,68 @@ class TestCLI(unittest.TestCase):
# Run test kernels without --arch flag
parser = osaca.create_parser()
# x86
kernel_x86 = 'kernel_x86.s'
kernel_x86 = "kernel_x86.s"
args = parser.parse_args([self._find_test_file(kernel_x86)])
output = StringIO()
osaca.run(args, output_file=output)
# AArch64
kernel_aarch64 = 'kernel_aarch64.s'
kernel_aarch64 = "kernel_aarch64.s"
args = parser.parse_args([self._find_test_file(kernel_aarch64)])
osaca.run(args, output_file=output)
def test_user_warnings(self):
parser = osaca.create_parser()
kernel = 'triad_x86_unmarked.s'
kernel = "triad_x86_unmarked.s"
args = parser.parse_args(
['--arch', 'csx', '--ignore-unknown', self._find_test_file(kernel)]
["--arch", "csx", "--ignore-unknown", self._find_test_file(kernel)]
)
output = StringIO()
osaca.run(args, output_file=output)
# WARNING for length
self.assertTrue(output.getvalue().count('WARNING') == 1)
self.assertTrue(output.getvalue().count("WARNING") == 1)
args = parser.parse_args(
['--lines', '100-199', '--ignore-unknown', self._find_test_file(kernel)]
["--lines", "100-199", "--ignore-unknown", self._find_test_file(kernel)]
)
output = StringIO()
osaca.run(args, output_file=output)
# WARNING for arch
self.assertTrue(output.getvalue().count('WARNING') == 1)
self.assertTrue(output.getvalue().count("WARNING") == 1)
def test_lines_arg(self):
# Run tests with --lines option
parser = osaca.create_parser()
kernel_x86 = 'triad_x86_iaca.s'
args_base = parser.parse_args(
['--arch', 'csx', self._find_test_file(kernel_x86)]
)
kernel_x86 = "triad_x86_iaca.s"
args_base = parser.parse_args(["--arch", "csx", self._find_test_file(kernel_x86)])
output_base = StringIO()
osaca.run(args_base, output_file=output_base)
output_base = output_base.getvalue().split('\n')[8:]
output_base = output_base.getvalue().split("\n")[8:]
args = []
args.append(parser.parse_args(
['--lines', '146-154', '--arch', 'csx', self._find_test_file(kernel_x86)]
))
args.append(parser.parse_args(
['--lines', '146:154', '--arch', 'csx', self._find_test_file(kernel_x86)]
))
args.append(parser.parse_args(
['--lines', '146,147:148,149-154', '--arch', 'csx', self._find_test_file(kernel_x86)]
))
args.append(
parser.parse_args(
["--lines", "146-154", "--arch", "csx", self._find_test_file(kernel_x86)]
)
)
args.append(
parser.parse_args(
["--lines", "146:154", "--arch", "csx", self._find_test_file(kernel_x86)]
)
)
args.append(
parser.parse_args(
[
"--lines",
"146,147:148,149-154",
"--arch",
"csx",
self._find_test_file(kernel_x86),
]
)
)
for a in args:
with self.subTest(params=a):
output = StringIO()
osaca.run(a, output_file=output)
self.assertEqual(output.getvalue().split('\n')[8:], output_base)
self.assertEqual(output.getvalue().split("\n")[8:], output_base)
##################
# Helper functions
@@ -231,23 +238,23 @@ class TestCLI(unittest.TestCase):
testdir = os.path.dirname(__file__)
name = os.path.join(
testdir,
'../examples',
"../examples",
kernel,
kernel + '.s.' + arch[:3].lower() + '.' + comp.lower() + '.s',
kernel + ".s." + arch[:3].lower() + "." + comp.lower() + ".s",
)
if kernel == 'j2d' and arch.lower() == 'csx':
name = name[:-1] + 'AVX.s'
if kernel == "j2d" and arch.lower() == "csx":
name = name[:-1] + "AVX.s"
assert os.path.exists(name)
return name
@staticmethod
def _find_test_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestCLI)
unittest.TextTestRunner(verbosity=2, buffer=True).run(suite)

View File

@@ -15,47 +15,47 @@ class TestDBInterface(unittest.TestCase):
@classmethod
def setUpClass(self):
sample_entry = {
'name': 'DoItRightAndDoItFast',
'operands': [
{'class': 'memory', 'offset': 'imd', 'base': 'gpr', 'index': 'gpr', 'scale': 8},
{'class': 'register', 'name': 'xmm'},
"name": "DoItRightAndDoItFast",
"operands": [
{"class": "memory", "offset": "imd", "base": "gpr", "index": "gpr", "scale": 8},
{"class": "register", "name": "xmm"},
],
'throughput': 1.25,
'latency': 125,
'uops': 6,
"throughput": 1.25,
"latency": 125,
"uops": 6,
}
self.entry_csx = sample_entry.copy()
self.entry_tx2 = sample_entry.copy()
self.entry_zen1 = sample_entry.copy()
# self.entry_csx['port_pressure'] = [1.25, 0, 1.25, 0.5, 0.5, 0.5, 0.5, 0, 1.25, 1.25, 0]
self.entry_csx['port_pressure'] = [[5, '0156'], [1, '23'], [1, ['2D', '3D']]]
self.entry_csx["port_pressure"] = [[5, "0156"], [1, "23"], [1, ["2D", "3D"]]]
# self.entry_tx2['port_pressure'] = [2.5, 2.5, 0, 0, 0.5, 0.5]
self.entry_tx2['port_pressure'] = [[5, '01'], [1, '45']]
del self.entry_tx2['operands'][1]['name']
self.entry_tx2['operands'][1]['prefix'] = 'x'
self.entry_tx2["port_pressure"] = [[5, "01"], [1, "45"]]
del self.entry_tx2["operands"][1]["name"]
self.entry_tx2["operands"][1]["prefix"] = "x"
# self.entry_zen1['port_pressure'] = [1, 1, 1, 1, 0, 1, 0, 0, 0, 0.5, 1, 0.5, 1]
self.entry_zen1['port_pressure'] = [[4, '0123'], [1, '4'], [1, '89'], [2, ['8D', '9D']]]
self.entry_zen1["port_pressure"] = [[4, "0123"], [1, "4"], [1, "89"], [2, ["8D", "9D"]]]
###########
# Tests
###########
def test_add_single_entry(self):
mm_csx = MachineModel('csx')
mm_tx2 = MachineModel('tx2')
mm_zen1 = MachineModel('zen1')
num_entries_csx = len(mm_csx['instruction_forms'])
num_entries_tx2 = len(mm_tx2['instruction_forms'])
num_entries_zen1 = len(mm_zen1['instruction_forms'])
mm_csx = MachineModel("csx")
mm_tx2 = MachineModel("tx2")
mm_zen1 = MachineModel("zen1")
num_entries_csx = len(mm_csx["instruction_forms"])
num_entries_tx2 = len(mm_tx2["instruction_forms"])
num_entries_zen1 = len(mm_zen1["instruction_forms"])
mm_csx.set_instruction_entry(self.entry_csx)
mm_tx2.set_instruction_entry(self.entry_tx2)
mm_zen1.set_instruction_entry({'name': 'empty_operation'})
mm_zen1.set_instruction_entry({"name": "empty_operation"})
num_entries_csx = len(mm_csx['instruction_forms']) - num_entries_csx
num_entries_tx2 = len(mm_tx2['instruction_forms']) - num_entries_tx2
num_entries_zen1 = len(mm_zen1['instruction_forms']) - num_entries_zen1
num_entries_csx = len(mm_csx["instruction_forms"]) - num_entries_csx
num_entries_tx2 = len(mm_tx2["instruction_forms"]) - num_entries_tx2
num_entries_zen1 = len(mm_zen1["instruction_forms"]) - num_entries_zen1
self.assertEqual(num_entries_csx, 1)
self.assertEqual(num_entries_tx2, 1)
@@ -64,76 +64,76 @@ class TestDBInterface(unittest.TestCase):
def test_invalid_add(self):
entry = {}
with self.assertRaises(KeyError):
MachineModel('csx').set_instruction_entry(entry)
MachineModel("csx").set_instruction_entry(entry)
with self.assertRaises(TypeError):
MachineModel('csx').set_instruction()
MachineModel("csx").set_instruction()
def test_sanity_check(self):
output = StringIO()
# non-verbose
sanity_check('csx', verbose=False, internet_check=False, output_file=output)
sanity_check('tx2', verbose=False, internet_check=False, output_file=output)
sanity_check('zen1', verbose=False, internet_check=False, output_file=output)
sanity_check("csx", verbose=False, internet_check=False, output_file=output)
sanity_check("tx2", verbose=False, internet_check=False, output_file=output)
sanity_check("zen1", verbose=False, internet_check=False, output_file=output)
# verbose
sanity_check('csx', verbose=True, internet_check=False, output_file=output)
sanity_check('tx2', verbose=True, internet_check=False, output_file=output)
sanity_check('zen1', verbose=True, internet_check=False, output_file=output)
sanity_check("csx", verbose=True, internet_check=False, output_file=output)
sanity_check("tx2", verbose=True, internet_check=False, output_file=output)
sanity_check("zen1", verbose=True, internet_check=False, output_file=output)
def test_ibench_import(self):
# only check import without dumping the DB file (takes too much time)
with open(self._find_file('ibench_import_x86.dat')) as input_file:
with open(self._find_file("ibench_import_x86.dat")) as input_file:
input_data = input_file.readlines()
entries = dbi._get_ibench_output(input_data, 'x86')
entries = dbi._get_ibench_output(input_data, "x86")
self.assertEqual(len(entries), 3)
for _, e in entries.items():
self.assertIsNotNone(e['throughput'])
self.assertIsNotNone(e['latency'])
with open(self._find_file('ibench_import_aarch64.dat')) as input_file:
self.assertIsNotNone(e["throughput"])
self.assertIsNotNone(e["latency"])
with open(self._find_file("ibench_import_aarch64.dat")) as input_file:
input_data = input_file.readlines()
entries = dbi._get_ibench_output(input_data, 'aarch64')
entries = dbi._get_ibench_output(input_data, "aarch64")
self.assertEqual(len(entries), 4)
for _, e in entries.items():
self.assertIsNotNone(e['throughput'])
self.assertIsNotNone(e['latency'])
self.assertIsNotNone(e["throughput"])
self.assertIsNotNone(e["latency"])
def test_asmbench_import(self):
# only check import without dumping the DB file (takes too much time)
with open(self._find_file('asmbench_import_x86.dat')) as input_file:
with open(self._find_file("asmbench_import_x86.dat")) as input_file:
input_data = input_file.readlines()
entries = dbi._get_asmbench_output(input_data, 'x86')
entries = dbi._get_asmbench_output(input_data, "x86")
self.assertEqual(len(entries), 3)
for _, e in entries.items():
self.assertIsNotNone(e['throughput'])
self.assertIsNotNone(e['latency'])
with open(self._find_file('asmbench_import_aarch64.dat')) as input_file:
self.assertIsNotNone(e["throughput"])
self.assertIsNotNone(e["latency"])
with open(self._find_file("asmbench_import_aarch64.dat")) as input_file:
input_data = input_file.readlines()
entries = dbi._get_asmbench_output(input_data, 'aarch64')
entries = dbi._get_asmbench_output(input_data, "aarch64")
self.assertEqual(len(entries), 4)
for _, e in entries.items():
self.assertIsNotNone(e['throughput'])
self.assertIsNotNone(e['latency'])
self.assertIsNotNone(e["throughput"])
self.assertIsNotNone(e["latency"])
# remove empty line => no import since broken format
del input_data[3]
entries = dbi._get_asmbench_output(input_data, 'aarch64')
entries = dbi._get_asmbench_output(input_data, "aarch64")
self.assertEqual(len(entries), 0)
with self.assertRaises(ValueError):
dbi.import_benchmark_output(
'csx', 'invalid_bench_type', self._find_file('asmbench_import_x86.dat')
"csx", "invalid_bench_type", self._find_file("asmbench_import_x86.dat")
)
with self.assertRaises(AssertionError):
dbi.import_benchmark_output('csx', 'ibench', 'invalid_file')
dbi.import_benchmark_output("csx", "ibench", "invalid_file")
def test_online_scraping(self):
# addpd -- suspicious instruction, normal URL
instr_1 = ['addpd', (True, '(r) (r,w)')]
instr_1 = ["addpd", (True, "(r) (r,w)")]
self.assertEqual(dbi._scrape_from_felixcloutier(instr_1[0]), instr_1[1])
# movpd -- not suspicious,
instr_2 = ['movapd', (False, '(r) (w)')]
instr_2 = ["movapd", (False, "(r) (w)")]
self.assertEqual(dbi._scrape_from_felixcloutier(instr_2[0]), instr_2[1])
# vfmadd132pd -- only in combined view with 213/231.
# No 2-operand version, therefore, empty string
instr_3 = ['vfmadd132pd', (True, '')]
instr_3 = ["vfmadd132pd", (True, "")]
self.assertEqual(dbi._scrape_from_felixcloutier(instr_3[0]), instr_3[1])
##################
@@ -142,11 +142,11 @@ class TestDBInterface(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestDBInterface)
unittest.TextTestRunner(verbosity=2, buffer=True).run(suite)

View File

@@ -13,7 +13,7 @@ from osaca.semantics import ArchSemantics, KernelDG, MachineModel
class TestFrontend(unittest.TestCase):
MODULE_DATA_DIR = os.path.join(
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/'
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), "osaca/data/"
)
@classmethod
@@ -21,26 +21,24 @@ class TestFrontend(unittest.TestCase):
# set up parser and kernels
self.parser_x86 = ParserX86ATT()
self.parser_AArch64 = ParserAArch64()
with open(self._find_file('kernel_x86.s')) as f:
with open(self._find_file("kernel_x86.s")) as f:
code_x86 = f.read()
with open(self._find_file('kernel_aarch64.s')) as f:
with open(self._find_file("kernel_aarch64.s")) as f:
code_AArch64 = f.read()
self.kernel_x86 = self.parser_x86.parse_file(code_x86)
self.kernel_AArch64 = self.parser_AArch64.parse_file(code_AArch64)
# set up machine models
self.machine_model_csx = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')
)
self.machine_model_tx2 = MachineModel(
arch='tx2'
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml")
)
self.machine_model_tx2 = MachineModel(arch="tx2")
self.semantics_csx = ArchSemantics(
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml')
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "isa/x86.yml")
)
self.semantics_tx2 = ArchSemantics(
self.machine_model_tx2,
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'),
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "isa/aarch64.yml"),
)
for i in range(len(self.kernel_x86)):
self.semantics_csx.assign_src_dst(self.kernel_x86[i])
@@ -57,23 +55,23 @@ class TestFrontend(unittest.TestCase):
with self.assertRaises(ValueError):
Frontend()
with self.assertRaises(ValueError):
Frontend(arch='csx', path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml'))
Frontend(arch="csx", path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml"))
with self.assertRaises(FileNotFoundError):
Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'THE_MACHINE.yml'))
Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "THE_MACHINE.yml"))
with self.assertRaises(FileNotFoundError):
Frontend(arch='THE_MACHINE')
Frontend(arch='zen1')
Frontend(arch="THE_MACHINE")
Frontend(arch="zen1")
def test_frontend_x86(self):
dg = KernelDG(self.kernel_x86, self.parser_x86, self.machine_model_csx)
fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml'))
fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml"))
fe.throughput_analysis(self.kernel_x86, show_cmnts=False)
fe.latency_analysis(dg.get_critical_path())
# TODO compare output with checked string
def test_frontend_AArch64(self):
dg = KernelDG(self.kernel_AArch64, self.parser_AArch64, self.machine_model_tx2)
fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml'))
fe = Frontend(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "tx2.yml"))
fe.full_analysis(self.kernel_AArch64, dg, verbose=True)
# TODO compare output with checked string
@@ -84,11 +82,11 @@ class TestFrontend(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestFrontend)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -18,9 +18,9 @@ class TestKerncraftAPI(unittest.TestCase):
# set up parser and kernels
self.parser_x86 = ParserX86ATT()
self.parser_AArch64 = ParserAArch64()
with open(self._find_file('triad_x86_iaca.s')) as f:
with open(self._find_file("triad_x86_iaca.s")) as f:
self.code_x86 = f.read()
with open(self._find_file('triad_arm_iaca.s')) as f:
with open(self._find_file("triad_arm_iaca.s")) as f:
self.code_AArch64 = f.read()
###########
@@ -28,23 +28,23 @@ class TestKerncraftAPI(unittest.TestCase):
###########
def test_kerncraft_API_x86(self):
kapi = KerncraftAPI('csx', self.code_x86)
kapi = KerncraftAPI("csx", self.code_x86)
kapi.create_output()
self.assertEqual(kapi.get_unmatched_instruction_ratio(), 0.0)
port_occupation = OrderedDict(
[
('0', 1.25),
('0DV', 0.0),
('1', 1.25),
('2', 2.0),
('2D', 1.5),
('3', 2.0),
('3D', 1.5),
('4', 1.0),
('5', 0.75),
('6', 0.75),
('7', 0.0),
("0", 1.25),
("0DV", 0.0),
("1", 1.25),
("2", 2.0),
("2D", 1.5),
("3", 2.0),
("3D", 1.5),
("4", 1.0),
("5", 0.75),
("6", 0.75),
("7", 0.0),
]
)
self.assertEqual(kapi.get_port_occupation_cycles(), port_occupation)
@@ -53,20 +53,20 @@ class TestKerncraftAPI(unittest.TestCase):
self.assertEqual(kapi.get_latency(), (1.0, 8.0))
def test_kerncraft_API_AArch64(self):
kapi = KerncraftAPI('tx2', self.code_AArch64)
kapi = KerncraftAPI("tx2", self.code_AArch64)
kapi.create_output()
self.assertEqual(kapi.get_unmatched_instruction_ratio(), 0.0)
port_occupation = OrderedDict(
[
('0', 34.0),
('0DV', 0.0),
('1', 34.0),
('1DV', 0.0),
('2', 3.0),
('3', 64.0),
('4', 64.0),
('5', 32.0),
("0", 34.0),
("0DV", 0.0),
("1", 34.0),
("1DV", 0.0),
("2", 3.0),
("3", 64.0),
("4", 64.0),
("5", 32.0),
]
)
self.assertEqual(kapi.get_port_occupation_cycles(), port_occupation)
@@ -81,11 +81,11 @@ class TestKerncraftAPI(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestKerncraftAPI)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -6,8 +6,12 @@ import os
import unittest
from collections import OrderedDict
from osaca.semantics import reduce_to_section, find_basic_blocks, find_jump_labels, \
find_basic_loop_bodies
from osaca.semantics import (
reduce_to_section,
find_basic_blocks,
find_jump_labels,
find_basic_loop_bodies,
)
from osaca.parser import ParserAArch64, ParserX86ATT
@@ -16,9 +20,9 @@ class TestMarkerUtils(unittest.TestCase):
def setUpClass(self):
self.parser_AArch = ParserAArch64()
self.parser_x86 = ParserX86ATT()
with open(self._find_file('triad_arm_iaca.s')) as f:
with open(self._find_file("triad_arm_iaca.s")) as f:
triad_code_arm = f.read()
with open(self._find_file('triad_x86_iaca.s')) as f:
with open(self._find_file("triad_x86_iaca.s")) as f:
triad_code_x86 = f.read()
self.parsed_AArch = self.parser_AArch.parse_file(triad_code_arm)
self.parsed_x86 = self.parser_x86.parse_file(triad_code_x86)
@@ -28,44 +32,44 @@ class TestMarkerUtils(unittest.TestCase):
#################
def test_marker_detection_AArch64(self):
kernel = reduce_to_section(self.parsed_AArch, 'AArch64')
kernel = reduce_to_section(self.parsed_AArch, "AArch64")
self.assertEqual(len(kernel), 138)
self.assertEqual(kernel[0].line_number, 307)
self.assertEqual(kernel[-1].line_number, 444)
def test_marker_detection_x86(self):
kernel = reduce_to_section(self.parsed_x86, 'x86')
kernel = reduce_to_section(self.parsed_x86, "x86")
self.assertEqual(len(kernel), 9)
self.assertEqual(kernel[0].line_number, 146)
self.assertEqual(kernel[-1].line_number, 154)
def test_marker_matching_AArch64(self):
# preparation
bytes_1_line = '.byte 213,3,32,31\n'
bytes_2_lines_1 = '.byte 213,3,32\n' + '.byte 31\n'
bytes_2_lines_2 = '.byte 213,3\n' + '.byte 32,31\n'
bytes_2_lines_3 = '.byte 213\n' + '.byte 3,32,31\n'
bytes_3_lines_1 = '.byte 213,3\n' + '.byte 32\n' + '.byte 31\n'
bytes_3_lines_2 = '.byte 213\n' + '.byte 3,32\n' + '.byte 31\n'
bytes_3_lines_3 = '.byte 213\n' + '.byte 3\n' + '.byte 32,31\n'
bytes_4_lines = '.byte 213\n' + '.byte 3\n' + '.byte 32\n' + '.byte 31\n'
mov_start_1 = 'mov x1, #111\n'
mov_start_2 = 'mov x1, 111 // should work as well\n'
mov_end_1 = 'mov x1, #222 // preferred way\n'
mov_end_2 = 'mov x1, 222\n'
bytes_1_line = ".byte 213,3,32,31\n"
bytes_2_lines_1 = ".byte 213,3,32\n" + ".byte 31\n"
bytes_2_lines_2 = ".byte 213,3\n" + ".byte 32,31\n"
bytes_2_lines_3 = ".byte 213\n" + ".byte 3,32,31\n"
bytes_3_lines_1 = ".byte 213,3\n" + ".byte 32\n" + ".byte 31\n"
bytes_3_lines_2 = ".byte 213\n" + ".byte 3,32\n" + ".byte 31\n"
bytes_3_lines_3 = ".byte 213\n" + ".byte 3\n" + ".byte 32,31\n"
bytes_4_lines = ".byte 213\n" + ".byte 3\n" + ".byte 32\n" + ".byte 31\n"
mov_start_1 = "mov x1, #111\n"
mov_start_2 = "mov x1, 111 // should work as well\n"
mov_end_1 = "mov x1, #222 // preferred way\n"
mov_end_2 = "mov x1, 222\n"
prologue = (
'mov x12, xzr\n'
+ '\tldp x9, x10, [sp, #16] // 8-byte Folded Reload\n'
+ ' .p2align 6\n'
"mov x12, xzr\n"
+ "\tldp x9, x10, [sp, #16] // 8-byte Folded Reload\n"
+ " .p2align 6\n"
)
kernel = (
'.LBB0_28:\n'
+ 'fmul v7.2d, v7.2d, v19.2d\n'
+ 'stp q0, q1, [x10, #-32]\n'
+ 'b.ne .LBB0_28\n'
".LBB0_28:\n"
+ "fmul v7.2d, v7.2d, v19.2d\n"
+ "stp q0, q1, [x10, #-32]\n"
+ "b.ne .LBB0_28\n"
)
epilogue = '.LBB0_29: // Parent Loop BB0_20 Depth=1\n' + 'bl dummy\n'
kernel_length = len(list(filter(None, kernel.split('\n'))))
epilogue = ".LBB0_29: // Parent Loop BB0_20 Depth=1\n" + "bl dummy\n"
kernel_length = len(list(filter(None, kernel.split("\n"))))
bytes_variations = [
bytes_1_line,
@@ -100,12 +104,12 @@ class TestMarkerUtils(unittest.TestCase):
bytes_end=bytes_var_2,
):
sample_parsed = self.parser_AArch.parse_file(sample_code)
sample_kernel = reduce_to_section(sample_parsed, 'AArch64')
sample_kernel = reduce_to_section(sample_parsed, "AArch64")
self.assertEqual(len(sample_kernel), kernel_length)
kernel_start = len(
list(
filter(
None, (prologue + mov_start_var + bytes_var_1).split('\n')
None, (prologue + mov_start_var + bytes_var_1).split("\n")
)
)
)
@@ -116,27 +120,27 @@ class TestMarkerUtils(unittest.TestCase):
def test_marker_matching_x86(self):
# preparation
bytes_1_line = '.byte 100,103,144\n'
bytes_2_lines_1 = '.byte 100,103\n' + '.byte 144\n'
bytes_2_lines_2 = '.byte 100\n' + '.byte 103,144\n'
bytes_1_line = ".byte 100,103,144\n"
bytes_2_lines_1 = ".byte 100,103\n" + ".byte 144\n"
bytes_2_lines_2 = ".byte 100\n" + ".byte 103,144\n"
bytes_3_lines = (
'.byte 100 # IACA MARKER UTILITY\n'
+ '.byte 103 # IACA MARKER UTILITY\n'
+ '.byte 144 # IACA MARKER UTILITY\n'
".byte 100 # IACA MARKER UTILITY\n"
+ ".byte 103 # IACA MARKER UTILITY\n"
+ ".byte 144 # IACA MARKER UTILITY\n"
)
mov_start_1 = 'movl $111, %ebx # IACA START\n'
mov_start_2 = 'mov $111, %ebx # IACA START\n'
mov_end_1 = 'movl $222, %ebx # IACA END\n'
mov_end_2 = 'mov $222, %ebx # IACA END\n'
prologue = 'movl -92(%rbp), %r11d\n' + 'movl $111, %ebx\n'
mov_start_1 = "movl $111, %ebx # IACA START\n"
mov_start_2 = "mov $111, %ebx # IACA START\n"
mov_end_1 = "movl $222, %ebx # IACA END\n"
mov_end_2 = "mov $222, %ebx # IACA END\n"
prologue = "movl -92(%rbp), %r11d\n" + "movl $111, %ebx\n"
kernel = (
'vfmadd132sd (%r15,%rcx,8), %xmm5, %xmm0\n'
+ 'vmovsd %xmm0, (%r14,%rcx,8)\n'
+ 'cmpl %ebx, %ecx\n'
+ 'jge .L8\n'
"vfmadd132sd (%r15,%rcx,8), %xmm5, %xmm0\n"
+ "vmovsd %xmm0, (%r14,%rcx,8)\n"
+ "cmpl %ebx, %ecx\n"
+ "jge .L8\n"
)
epilogue = '.LE9:\t\t#12.2\n' 'call dummy\n'
kernel_length = len(list(filter(None, kernel.split('\n'))))
epilogue = ".LE9:\t\t#12.2\n" "call dummy\n"
kernel_length = len(list(filter(None, kernel.split("\n"))))
bytes_variations = [bytes_1_line, bytes_2_lines_1, bytes_2_lines_2, bytes_3_lines]
mov_start_variations = [mov_start_1, mov_start_2]
@@ -162,12 +166,12 @@ class TestMarkerUtils(unittest.TestCase):
bytes_end=bytes_var_2,
):
sample_parsed = self.parser_x86.parse_file(sample_code)
sample_kernel = reduce_to_section(sample_parsed, 'x86')
sample_kernel = reduce_to_section(sample_parsed, "x86")
self.assertEqual(len(sample_kernel), kernel_length)
kernel_start = len(
list(
filter(
None, (prologue + mov_start_var + bytes_var_1).split('\n')
None, (prologue + mov_start_var + bytes_var_1).split("\n")
)
)
)
@@ -177,171 +181,250 @@ class TestMarkerUtils(unittest.TestCase):
self.assertEqual(sample_kernel, parsed_kernel)
def test_marker_special_cases_AArch(self):
bytes_line = '.byte 213,3,32,31\n'
start_marker = 'mov x1, #111\n' + bytes_line
end_marker = 'mov x1, #222\n' + bytes_line
prologue = (
'dup v0.2d, x14\n'
'neg x9, x9\n'
'.p2align 6\n')
bytes_line = ".byte 213,3,32,31\n"
start_marker = "mov x1, #111\n" + bytes_line
end_marker = "mov x1, #222\n" + bytes_line
prologue = "dup v0.2d, x14\n" "neg x9, x9\n" ".p2align 6\n"
kernel = (
'.LBB0_28:\n'
+ 'fmul v7.2d, v7.2d, v19.2d\n'
+ 'stp q0, q1, [x10, #-32]\n'
+ 'b.ne .LBB0_28\n')
epilogue = (
'.LBB0_29: // Parent Loop BB0_20 Depth=1\n'
'bl dummy\n')
".LBB0_28:\n"
+ "fmul v7.2d, v7.2d, v19.2d\n"
+ "stp q0, q1, [x10, #-32]\n"
+ "b.ne .LBB0_28\n"
)
epilogue = ".LBB0_29: // Parent Loop BB0_20 Depth=1\n" "bl dummy\n"
samples = [
# (test name,
# ignored prologue, section to be extraced, ignored epilogue)
("markers",
prologue + start_marker, kernel, end_marker + epilogue),
("marker at file start",
start_marker, kernel, end_marker + epilogue),
("no start marker",
'', prologue + kernel, end_marker + epilogue),
("marker at file end",
prologue + start_marker, kernel, end_marker),
("no end marker",
prologue + start_marker, kernel + epilogue, ''),
("empty kernel",
prologue + start_marker, '', end_marker + epilogue),
("markers", prologue + start_marker, kernel, end_marker + epilogue),
("marker at file start", start_marker, kernel, end_marker + epilogue),
("no start marker", "", prologue + kernel, end_marker + epilogue),
("marker at file end", prologue + start_marker, kernel, end_marker),
("no end marker", prologue + start_marker, kernel + epilogue, ""),
("empty kernel", prologue + start_marker, "", end_marker + epilogue),
]
for test_name, pro, kernel, epi in samples:
code = pro + kernel + epi
parsed = self.parser_AArch.parse_file(code)
test_kernel = reduce_to_section(parsed, 'AArch64')
test_kernel = reduce_to_section(parsed, "AArch64")
if kernel:
kernel_length = len(kernel.strip().split('\n'))
kernel_length = len(kernel.strip().split("\n"))
else:
kernel_length = 0
self.assertEqual(
len(test_kernel), kernel_length,
msg="Invalid exctracted kernel length on {!r} sample".format(test_name))
len(test_kernel),
kernel_length,
msg="Invalid exctracted kernel length on {!r} sample".format(test_name),
)
if pro:
kernel_start = len((pro).strip().split('\n'))
kernel_start = len((pro).strip().split("\n"))
else:
kernel_start = 0
parsed_kernel = self.parser_AArch.parse_file(kernel, start_line=kernel_start)
self.assertEqual(
test_kernel, parsed_kernel,
msg="Invalid exctracted kernel on {!r}".format(test_name))
test_kernel,
parsed_kernel,
msg="Invalid exctracted kernel on {!r}".format(test_name),
)
def test_marker_special_cases_x86(self):
bytes_line = (
'.byte 100\n'
'.byte 103\n'
'.byte 144\n')
start_marker = 'movl $111, %ebx\n' + bytes_line
end_marker = 'movl $222, %ebx\n' + bytes_line
prologue = (
'movl -88(%rbp), %r10d\n'
'xorl %r11d, %r11d\n'
'.p2align 4,,10\n')
bytes_line = ".byte 100\n" ".byte 103\n" ".byte 144\n"
start_marker = "movl $111, %ebx\n" + bytes_line
end_marker = "movl $222, %ebx\n" + bytes_line
prologue = "movl -88(%rbp), %r10d\n" "xorl %r11d, %r11d\n" ".p2align 4,,10\n"
kernel = (
'.L3: #L3\n'
'vmovsd .LC1(%rip), %xmm0\n'
'vmovsd %xmm0, (%r15,%rcx,8)\n'
'cmpl %ecx, %ebx\n'
'jle .L3\n')
epilogue = (
'leaq -56(%rbp), %rsi\n'
'movl %r10d, -88(%rbp)\n'
'call timing\n')
".L3: #L3\n"
"vmovsd .LC1(%rip), %xmm0\n"
"vmovsd %xmm0, (%r15,%rcx,8)\n"
"cmpl %ecx, %ebx\n"
"jle .L3\n"
)
epilogue = "leaq -56(%rbp), %rsi\n" "movl %r10d, -88(%rbp)\n" "call timing\n"
samples = [
# (test name,
# ignored prologue, section to be extraced, ignored epilogue)
("markers",
prologue + start_marker, kernel, end_marker + epilogue),
("marker at file start",
start_marker, kernel, end_marker + epilogue),
("no start marker",
'', prologue + kernel, end_marker + epilogue),
("marker at file end",
prologue + start_marker, kernel, end_marker),
("no end marker",
prologue + start_marker, kernel + epilogue, ''),
("empty kernel",
prologue + start_marker, '', end_marker + epilogue),
("markers", prologue + start_marker, kernel, end_marker + epilogue),
("marker at file start", start_marker, kernel, end_marker + epilogue),
("no start marker", "", prologue + kernel, end_marker + epilogue),
("marker at file end", prologue + start_marker, kernel, end_marker),
("no end marker", prologue + start_marker, kernel + epilogue, ""),
("empty kernel", prologue + start_marker, "", end_marker + epilogue),
]
for test_name, pro, kernel, epi in samples:
code = pro + kernel + epi
parsed = self.parser_x86.parse_file(code)
test_kernel = reduce_to_section(parsed, 'x86')
test_kernel = reduce_to_section(parsed, "x86")
if kernel:
kernel_length = len(kernel.strip().split('\n'))
kernel_length = len(kernel.strip().split("\n"))
else:
kernel_length = 0
self.assertEqual(
len(test_kernel), kernel_length,
msg="Invalid exctracted kernel length on {!r} sample".format(test_name))
len(test_kernel),
kernel_length,
msg="Invalid exctracted kernel length on {!r} sample".format(test_name),
)
if pro:
kernel_start = len((pro).strip().split('\n'))
kernel_start = len((pro).strip().split("\n"))
else:
kernel_start = 0
parsed_kernel = self.parser_x86.parse_file(kernel, start_line=kernel_start)
self.assertEqual(
test_kernel, parsed_kernel,
msg="Invalid exctracted kernel on {!r}".format(test_name))
test_kernel,
parsed_kernel,
msg="Invalid exctracted kernel on {!r}".format(test_name),
)
def test_find_jump_labels(self):
self.assertEqual(find_jump_labels(self.parsed_x86),
OrderedDict([('.LFB24', 10), ('.L4', 65), ('.L3', 79), ('.L2', 102),
('.L13', 111), ('.L12', 120), ('.L6', 132), ('.L10', 145),
('.L9', 161), ('.L8', 183), ('.L15', 252), ('.L26', 256),
('.L14', 259), ('.LFB25', 277), ('.L28', 289)]))
self.assertEqual(
find_jump_labels(self.parsed_x86),
OrderedDict(
[
(".LFB24", 10),
(".L4", 65),
(".L3", 79),
(".L2", 102),
(".L13", 111),
(".L12", 120),
(".L6", 132),
(".L10", 145),
(".L9", 161),
(".L8", 183),
(".L15", 252),
(".L26", 256),
(".L14", 259),
(".LFB25", 277),
(".L28", 289),
]
),
)
self.assertEqual(
find_jump_labels(self.parsed_AArch),
OrderedDict([('triad', 18), ('.LBB0_3', 71), ('.LBB0_4', 76), ('.LBB0_5', 84),
('.LBB0_7', 92), ('.LBB0_8', 95), ('.LBB0_9', 106), ('.LBB0_11', 118),
('.LBB0_12', 133), ('.LBB0_14', 177), ('.LBB0_15', 190),
('.LBB0_16', 205), ('.LBB0_17', 208), ('.LBB0_18', 221),
('.LBB0_19', 228), ('.LBB0_20', 260), ('.LBB0_22', 272),
('.LBB0_24', 283), ('.LBB0_26', 290), ('.LBB0_28', 298),
('.LBB0_29', 306), ('.LBB0_31', 448), ('.LBB0_32', 458),
('.LBB0_33', 480), ('.LBB0_34', 484), ('.LBB0_35', 493),
('.LBB0_36', 504), ('.LBB0_37', 508), ('.LBB0_38', 518),
('main', 574)]))
OrderedDict(
[
("triad", 18),
(".LBB0_3", 71),
(".LBB0_4", 76),
(".LBB0_5", 84),
(".LBB0_7", 92),
(".LBB0_8", 95),
(".LBB0_9", 106),
(".LBB0_11", 118),
(".LBB0_12", 133),
(".LBB0_14", 177),
(".LBB0_15", 190),
(".LBB0_16", 205),
(".LBB0_17", 208),
(".LBB0_18", 221),
(".LBB0_19", 228),
(".LBB0_20", 260),
(".LBB0_22", 272),
(".LBB0_24", 283),
(".LBB0_26", 290),
(".LBB0_28", 298),
(".LBB0_29", 306),
(".LBB0_31", 448),
(".LBB0_32", 458),
(".LBB0_33", 480),
(".LBB0_34", 484),
(".LBB0_35", 493),
(".LBB0_36", 504),
(".LBB0_37", 508),
(".LBB0_38", 518),
("main", 574),
]
),
)
def test_find_basic_blocks(self):
self.assertEqual(
[(k, v[0]['line_number'], v[-1]['line_number'])
for k, v in find_basic_blocks(self.parsed_x86).items()],
[('.LFB24', 11, 56), ('.L4', 66, 74), ('.L3', 80, 89), ('.L2', 103, 112),
('.L13', 112, 121), ('.L12', 121, 125), ('.L6', 133, 135), ('.L10', 146, 154),
('.L9', 162, 170), ('.L8', 184, 187), ('.L15', 253, 256), ('.L26', 257, 259),
('.L14', 260, 262), ('.LFB25', 278, 290), ('.L28', 290, 300)])
[
(k, v[0]["line_number"], v[-1]["line_number"])
for k, v in find_basic_blocks(self.parsed_x86).items()
],
[
(".LFB24", 11, 56),
(".L4", 66, 74),
(".L3", 80, 89),
(".L2", 103, 112),
(".L13", 112, 121),
(".L12", 121, 125),
(".L6", 133, 135),
(".L10", 146, 154),
(".L9", 162, 170),
(".L8", 184, 187),
(".L15", 253, 256),
(".L26", 257, 259),
(".L14", 260, 262),
(".LFB25", 278, 290),
(".L28", 290, 300),
],
)
self.assertEqual(
[(k, v[0]['line_number'], v[-1]['line_number'])
for k, v in find_basic_blocks(self.parsed_AArch).items()],
[('triad', 19, 64), ('.LBB0_3', 72, 77), ('.LBB0_4', 77, 83), ('.LBB0_5', 85, 89),
('.LBB0_7', 93, 95), ('.LBB0_8', 96, 105), ('.LBB0_9', 107, 114),
('.LBB0_11', 119, 134), ('.LBB0_12', 134, 173), ('.LBB0_14', 178, 191),
('.LBB0_15', 191, 205), ('.LBB0_16', 206, 208), ('.LBB0_17', 209, 222),
('.LBB0_18', 222, 228), ('.LBB0_19', 229, 261), ('.LBB0_20', 261, 269),
('.LBB0_22', 273, 280), ('.LBB0_24', 284, 286), ('.LBB0_26', 291, 293),
('.LBB0_28', 299, 307), ('.LBB0_29', 307, 444), ('.LBB0_31', 449, 459),
('.LBB0_32', 459, 480), ('.LBB0_33', 481, 484), ('.LBB0_34', 485, 494),
('.LBB0_35', 494, 504), ('.LBB0_36', 505, 508), ('.LBB0_37', 509, 518),
('.LBB0_38', 519, 568), ('main', 575, 590)])
[
(k, v[0]["line_number"], v[-1]["line_number"])
for k, v in find_basic_blocks(self.parsed_AArch).items()
],
[
("triad", 19, 64),
(".LBB0_3", 72, 77),
(".LBB0_4", 77, 83),
(".LBB0_5", 85, 89),
(".LBB0_7", 93, 95),
(".LBB0_8", 96, 105),
(".LBB0_9", 107, 114),
(".LBB0_11", 119, 134),
(".LBB0_12", 134, 173),
(".LBB0_14", 178, 191),
(".LBB0_15", 191, 205),
(".LBB0_16", 206, 208),
(".LBB0_17", 209, 222),
(".LBB0_18", 222, 228),
(".LBB0_19", 229, 261),
(".LBB0_20", 261, 269),
(".LBB0_22", 273, 280),
(".LBB0_24", 284, 286),
(".LBB0_26", 291, 293),
(".LBB0_28", 299, 307),
(".LBB0_29", 307, 444),
(".LBB0_31", 449, 459),
(".LBB0_32", 459, 480),
(".LBB0_33", 481, 484),
(".LBB0_34", 485, 494),
(".LBB0_35", 494, 504),
(".LBB0_36", 505, 508),
(".LBB0_37", 509, 518),
(".LBB0_38", 519, 568),
("main", 575, 590),
],
)
def test_find_basic_loop_body(self):
self.assertEqual(
[(k, v[0]['line_number'], v[-1]['line_number'])
for k, v in find_basic_loop_bodies(self.parsed_x86).items()],
[('.L4', 66, 74), ('.L10', 146, 154), ('.L28', 290, 300)])
[
(k, v[0]["line_number"], v[-1]["line_number"])
for k, v in find_basic_loop_bodies(self.parsed_x86).items()
],
[(".L4", 66, 74), (".L10", 146, 154), (".L28", 290, 300)],
)
self.assertEqual(
[(k, v[0]['line_number'], v[-1]['line_number'])
for k, v in find_basic_loop_bodies(self.parsed_AArch).items()],
[('.LBB0_12', 134, 173), ('.LBB0_15', 191, 205), ('.LBB0_18', 222, 228),
('.LBB0_29', 307, 444), ('.LBB0_32', 459, 480), ('.LBB0_35', 494, 504)])
[
(k, v[0]["line_number"], v[-1]["line_number"])
for k, v in find_basic_loop_bodies(self.parsed_AArch).items()
],
[
(".LBB0_12", 134, 173),
(".LBB0_15", 191, 205),
(".LBB0_18", 222, 228),
(".LBB0_29", 307, 444),
(".LBB0_32", 459, 480),
(".LBB0_35", 494, 504),
],
)
##################
# Helper functions
@@ -350,11 +433,11 @@ class TestMarkerUtils(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestMarkerUtils)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -15,7 +15,7 @@ class TestParserAArch64(unittest.TestCase):
@classmethod
def setUpClass(self):
self.parser = ParserAArch64()
with open(self._find_file('triad_arm_iaca.s')) as f:
with open(self._find_file("triad_arm_iaca.s")) as f:
self.triad_code = f.read()
##################
@@ -23,63 +23,61 @@ class TestParserAArch64(unittest.TestCase):
##################
def test_comment_parser(self):
self.assertEqual(self._get_comment(self.parser, '// some comments'), 'some comments')
self.assertEqual(self._get_comment(self.parser, "// some comments"), "some comments")
self.assertEqual(self._get_comment(self.parser, "\t\t//AA BB CC \t end \t"), "AA BB CC end")
self.assertEqual(
self._get_comment(self.parser, '\t\t//AA BB CC \t end \t'), 'AA BB CC end'
)
self.assertEqual(
self._get_comment(self.parser, '\t//// comment //// comment'),
'// comment //// comment',
self._get_comment(self.parser, "\t//// comment //// comment"),
"// comment //// comment",
)
def test_label_parser(self):
self.assertEqual(self._get_label(self.parser, 'main:').name, 'main')
self.assertEqual(self._get_label(self.parser, '..B1.10:').name, '..B1.10')
self.assertEqual(self._get_label(self.parser, '.2.3_2_pack.3:').name, '.2.3_2_pack.3')
self.assertEqual(self._get_label(self.parser, '.L1:\t\t\t//label1').name, '.L1')
self.assertEqual(self._get_label(self.parser, "main:").name, "main")
self.assertEqual(self._get_label(self.parser, "..B1.10:").name, "..B1.10")
self.assertEqual(self._get_label(self.parser, ".2.3_2_pack.3:").name, ".2.3_2_pack.3")
self.assertEqual(self._get_label(self.parser, ".L1:\t\t\t//label1").name, ".L1")
self.assertEqual(
' '.join(self._get_label(self.parser, '.L1:\t\t\t//label1').comment), 'label1'
" ".join(self._get_label(self.parser, ".L1:\t\t\t//label1").comment), "label1"
)
with self.assertRaises(ParseException):
self._get_label(self.parser, '\t.cfi_startproc')
self._get_label(self.parser, "\t.cfi_startproc")
def test_directive_parser(self):
self.assertEqual(self._get_directive(self.parser, '\t.text').name, 'text')
self.assertEqual(len(self._get_directive(self.parser, '\t.text').parameters), 0)
self.assertEqual(self._get_directive(self.parser, '\t.align\t16,0x90').name, 'align')
self.assertEqual(len(self._get_directive(self.parser, '\t.align\t16,0x90').parameters), 2)
self.assertEqual(self._get_directive(self.parser, "\t.text").name, "text")
self.assertEqual(len(self._get_directive(self.parser, "\t.text").parameters), 0)
self.assertEqual(self._get_directive(self.parser, "\t.align\t16,0x90").name, "align")
self.assertEqual(len(self._get_directive(self.parser, "\t.align\t16,0x90").parameters), 2)
self.assertEqual(
self._get_directive(self.parser, '\t.align\t16,0x90').parameters[1], '0x90'
self._get_directive(self.parser, "\t.align\t16,0x90").parameters[1], "0x90"
)
self.assertEqual(
self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[
'name'
self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[
"name"
],
'byte',
"byte",
)
self.assertEqual(
self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[
'parameters'
self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[
"parameters"
][2],
'144',
"144",
)
self.assertEqual(
' '.join(
self._get_directive(self.parser, ' .byte 100,103,144 //IACA START')[
'comment'
" ".join(
self._get_directive(self.parser, " .byte 100,103,144 //IACA START")[
"comment"
]
),
'IACA START',
"IACA START",
)
def test_parse_instruction(self):
instr1 = '\t\tvcvt.F32.S32 w1, w2\t\t\t//12.27'
instr2 = 'b.lo ..B1.4 \t'
instr3 = ' mov x2,#0x222 //NOT IACA END'
instr4 = 'str x28, [sp, x1, lsl #4] //12.9'
instr5 = 'ldr x0, [x0, #:got_lo12:q2c]'
instr6 = 'adrp x0, :got:visited'
instr7 = 'fadd v17.2d, v16.2d, v1.2d'
instr1 = "\t\tvcvt.F32.S32 w1, w2\t\t\t//12.27"
instr2 = "b.lo ..B1.4 \t"
instr3 = " mov x2,#0x222 //NOT IACA END"
instr4 = "str x28, [sp, x1, lsl #4] //12.9"
instr5 = "ldr x0, [x0, #:got_lo12:q2c]"
instr6 = "adrp x0, :got:visited"
instr7 = "fadd v17.2d, v16.2d, v1.2d"
parsed_1 = self.parser.parse_instruction(instr1)
parsed_2 = self.parser.parse_instruction(instr2)
@@ -89,198 +87,196 @@ class TestParserAArch64(unittest.TestCase):
parsed_6 = self.parser.parse_instruction(instr6)
parsed_7 = self.parser.parse_instruction(instr7)
self.assertEqual(parsed_1.instruction, 'vcvt.F32.S32')
self.assertEqual(parsed_1.operands[0].register.name, '1')
self.assertEqual(parsed_1.operands[0].register.prefix, 'w')
self.assertEqual(parsed_1.operands[1].register.name, '2')
self.assertEqual(parsed_1.operands[1].register.prefix, 'w')
self.assertEqual(parsed_1.comment, '12.27')
self.assertEqual(parsed_1.instruction, "vcvt.F32.S32")
self.assertEqual(parsed_1.operands[0].register.name, "1")
self.assertEqual(parsed_1.operands[0].register.prefix, "w")
self.assertEqual(parsed_1.operands[1].register.name, "2")
self.assertEqual(parsed_1.operands[1].register.prefix, "w")
self.assertEqual(parsed_1.comment, "12.27")
self.assertEqual(parsed_2.instruction, 'b.lo')
self.assertEqual(parsed_2.operands[0].identifier.name, '..B1.4')
self.assertEqual(parsed_2.instruction, "b.lo")
self.assertEqual(parsed_2.operands[0].identifier.name, "..B1.4")
self.assertEqual(len(parsed_2.operands), 1)
self.assertIsNone(parsed_2.comment)
self.assertEqual(parsed_3.instruction, 'mov')
self.assertEqual(parsed_3.operands[0].register.name, '2')
self.assertEqual(parsed_3.operands[0].register.prefix, 'x')
self.assertEqual(parsed_3.operands[1].immediate.value, '0x222')
self.assertEqual(parsed_3.comment, 'NOT IACA END')
self.assertEqual(parsed_3.instruction, "mov")
self.assertEqual(parsed_3.operands[0].register.name, "2")
self.assertEqual(parsed_3.operands[0].register.prefix, "x")
self.assertEqual(parsed_3.operands[1].immediate.value, "0x222")
self.assertEqual(parsed_3.comment, "NOT IACA END")
self.assertEqual(parsed_4.instruction, 'str')
self.assertEqual(parsed_4.instruction, "str")
self.assertIsNone(parsed_4.operands[1].memory.offset)
self.assertEqual(parsed_4.operands[1].memory.base.name, 'sp')
self.assertEqual(parsed_4.operands[1].memory.base.prefix, 'x')
self.assertEqual(parsed_4.operands[1].memory.index.name, '1')
self.assertEqual(parsed_4.operands[1].memory.index.prefix, 'x')
self.assertEqual(parsed_4.operands[1].memory.base.name, "sp")
self.assertEqual(parsed_4.operands[1].memory.base.prefix, "x")
self.assertEqual(parsed_4.operands[1].memory.index.name, "1")
self.assertEqual(parsed_4.operands[1].memory.index.prefix, "x")
self.assertEqual(parsed_4.operands[1].memory.scale, 16)
self.assertEqual(parsed_4.operands[0].register.name, '28')
self.assertEqual(parsed_4.operands[0].register.prefix, 'x')
self.assertEqual(parsed_4.comment, '12.9')
self.assertEqual(parsed_4.operands[0].register.name, "28")
self.assertEqual(parsed_4.operands[0].register.prefix, "x")
self.assertEqual(parsed_4.comment, "12.9")
self.assertEqual(parsed_5.instruction, 'ldr')
self.assertEqual(parsed_5.operands[0].register.name, '0')
self.assertEqual(parsed_5.operands[0].register.prefix, 'x')
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, 'q2c')
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.relocation,
':got_lo12:')
self.assertEqual(parsed_5.operands[1].memory.base.name, '0')
self.assertEqual(parsed_5.operands[1].memory.base.prefix, 'x')
self.assertEqual(parsed_5.instruction, "ldr")
self.assertEqual(parsed_5.operands[0].register.name, "0")
self.assertEqual(parsed_5.operands[0].register.prefix, "x")
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, "q2c")
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.relocation, ":got_lo12:")
self.assertEqual(parsed_5.operands[1].memory.base.name, "0")
self.assertEqual(parsed_5.operands[1].memory.base.prefix, "x")
self.assertIsNone(parsed_5.operands[1].memory.index)
self.assertEqual(parsed_5.operands[1].memory.scale, 1)
self.assertEqual(parsed_6.instruction, 'adrp')
self.assertEqual(parsed_6.operands[0].register.name, '0')
self.assertEqual(parsed_6.operands[0].register.prefix, 'x')
self.assertEqual(parsed_6.operands[1].identifier.relocation, ':got:')
self.assertEqual(parsed_6.operands[1].identifier.name, 'visited')
self.assertEqual(parsed_6.instruction, "adrp")
self.assertEqual(parsed_6.operands[0].register.name, "0")
self.assertEqual(parsed_6.operands[0].register.prefix, "x")
self.assertEqual(parsed_6.operands[1].identifier.relocation, ":got:")
self.assertEqual(parsed_6.operands[1].identifier.name, "visited")
self.assertEqual(parsed_7.instruction, 'fadd')
self.assertEqual(parsed_7.operands[0].register.name, '17')
self.assertEqual(parsed_7.operands[0].register.prefix, 'v')
self.assertEqual(parsed_7.operands[0].register.lanes, '2')
self.assertEqual(parsed_7.operands[0].register.shape, 'd')
self.assertEqual(self.parser.get_full_reg_name(parsed_7.operands[2].register),
'v1.2d')
self.assertEqual(parsed_7.instruction, "fadd")
self.assertEqual(parsed_7.operands[0].register.name, "17")
self.assertEqual(parsed_7.operands[0].register.prefix, "v")
self.assertEqual(parsed_7.operands[0].register.lanes, "2")
self.assertEqual(parsed_7.operands[0].register.shape, "d")
self.assertEqual(self.parser.get_full_reg_name(parsed_7.operands[2].register), "v1.2d")
def test_parse_line(self):
line_comment = '// -- Begin main'
line_label = '.LBB0_1: // =>This Inner Loop Header: Depth=1'
line_directive = '.cfi_def_cfa w29, -16'
line_instruction = 'ldr s0, [x11, w10, sxtw #2] // = <<2'
line_prefetch = 'prfm pldl1keep, [x26, #2048] //HPL'
line_preindexed = 'stp x29, x30, [sp, #-16]!'
line_postindexed = 'ldp q2, q3, [x11], #64'
line_5_operands = 'fcmla z26.d, p0/m, z29.d, z21.d, #90'
line_comment = "// -- Begin main"
line_label = ".LBB0_1: // =>This Inner Loop Header: Depth=1"
line_directive = ".cfi_def_cfa w29, -16"
line_instruction = "ldr s0, [x11, w10, sxtw #2] // = <<2"
line_prefetch = "prfm pldl1keep, [x26, #2048] //HPL"
line_preindexed = "stp x29, x30, [sp, #-16]!"
line_postindexed = "ldp q2, q3, [x11], #64"
line_5_operands = "fcmla z26.d, p0/m, z29.d, z21.d, #90"
instruction_form_1 = {
'instruction': None,
'operands': [],
'directive': None,
'comment': '-- Begin main',
'label': None,
'line': '// -- Begin main',
'line_number': 1,
"instruction": None,
"operands": [],
"directive": None,
"comment": "-- Begin main",
"label": None,
"line": "// -- Begin main",
"line_number": 1,
}
instruction_form_2 = {
'instruction': None,
'operands': [],
'directive': None,
'comment': '=>This Inner Loop Header: Depth=1',
'label': '.LBB0_1',
'line': '.LBB0_1: // =>This Inner Loop Header: Depth=1',
'line_number': 2,
"instruction": None,
"operands": [],
"directive": None,
"comment": "=>This Inner Loop Header: Depth=1",
"label": ".LBB0_1",
"line": ".LBB0_1: // =>This Inner Loop Header: Depth=1",
"line_number": 2,
}
instruction_form_3 = {
'instruction': None,
'operands': [],
'directive': {'name': 'cfi_def_cfa', 'parameters': ['w29', '-16']},
'comment': None,
'label': None,
'line': '.cfi_def_cfa w29, -16',
'line_number': 3,
"instruction": None,
"operands": [],
"directive": {"name": "cfi_def_cfa", "parameters": ["w29", "-16"]},
"comment": None,
"label": None,
"line": ".cfi_def_cfa w29, -16",
"line_number": 3,
}
instruction_form_4 = {
'instruction': 'ldr',
'operands': [
{'register': {'prefix': 's', 'name': '0'}},
"instruction": "ldr",
"operands": [
{"register": {"prefix": "s", "name": "0"}},
{
'memory': {
'offset': None,
'base': {'prefix': 'x', 'name': '11'},
'index': {
'prefix': 'w',
'name': '10',
'shift_op': 'sxtw',
'immediate': {'value': '2'},
'shift': [{'value': '2'}],
"memory": {
"offset": None,
"base": {"prefix": "x", "name": "11"},
"index": {
"prefix": "w",
"name": "10",
"shift_op": "sxtw",
"immediate": {"value": "2"},
"shift": [{"value": "2"}],
},
'scale': 4,
"scale": 4,
}
},
],
'directive': None,
'comment': '= <<2',
'label': None,
'line': 'ldr s0, [x11, w10, sxtw #2] // = <<2',
'line_number': 4,
"directive": None,
"comment": "= <<2",
"label": None,
"line": "ldr s0, [x11, w10, sxtw #2] // = <<2",
"line_number": 4,
}
instruction_form_5 = {
'instruction': 'prfm',
'operands': [
{'prfop': {'type': ['PLD'], 'target': ['L1'], 'policy': ['KEEP']}},
"instruction": "prfm",
"operands": [
{"prfop": {"type": ["PLD"], "target": ["L1"], "policy": ["KEEP"]}},
{
'memory': {
'offset': {'value': '2048'},
'base': {'prefix': 'x', 'name': '26'},
'index': None,
'scale': 1,
"memory": {
"offset": {"value": "2048"},
"base": {"prefix": "x", "name": "26"},
"index": None,
"scale": 1,
}
},
],
'directive': None,
'comment': 'HPL',
'label': None,
'line': 'prfm pldl1keep, [x26, #2048] //HPL',
'line_number': 5,
"directive": None,
"comment": "HPL",
"label": None,
"line": "prfm pldl1keep, [x26, #2048] //HPL",
"line_number": 5,
}
instruction_form_6 = {
'instruction': 'stp',
'operands': [
{'register': {'prefix': 'x', 'name': '29'}},
{'register': {'prefix': 'x', 'name': '30'}},
"instruction": "stp",
"operands": [
{"register": {"prefix": "x", "name": "29"}},
{"register": {"prefix": "x", "name": "30"}},
{
'memory': {
'offset': {'value': '-16'},
'base': {'name': 'sp', 'prefix': 'x'},
'index': None,
'scale': 1,
'pre_indexed': True,
"memory": {
"offset": {"value": "-16"},
"base": {"name": "sp", "prefix": "x"},
"index": None,
"scale": 1,
"pre_indexed": True,
}
},
],
'directive': None,
'comment': None,
'label': None,
'line': 'stp x29, x30, [sp, #-16]!',
'line_number': 6,
"directive": None,
"comment": None,
"label": None,
"line": "stp x29, x30, [sp, #-16]!",
"line_number": 6,
}
instruction_form_7 = {
'instruction': 'ldp',
'operands': [
{'register': {'prefix': 'q', 'name': '2'}},
{'register': {'prefix': 'q', 'name': '3'}},
"instruction": "ldp",
"operands": [
{"register": {"prefix": "q", "name": "2"}},
{"register": {"prefix": "q", "name": "3"}},
{
'memory': {
'offset': None,
'base': {'prefix': 'x', 'name': '11'},
'index': None,
'scale': 1,
'post_indexed': {'value': '64'},
"memory": {
"offset": None,
"base": {"prefix": "x", "name": "11"},
"index": None,
"scale": 1,
"post_indexed": {"value": "64"},
}
},
],
'directive': None,
'comment': None,
'label': None,
'line': 'ldp q2, q3, [x11], #64',
'line_number': 7,
"directive": None,
"comment": None,
"label": None,
"line": "ldp q2, q3, [x11], #64",
"line_number": 7,
}
instruction_form_8 = {
'instruction': 'fcmla',
'operands': [
{'register': {'prefix': 'z', 'name': '26', 'shape': 'd'}},
{'register': {'prefix': 'p', 'name': '0', 'predication': 'm'}},
{'register': {'prefix': 'z', 'name': '29', 'shape': 'd'}},
{'register': {'prefix': 'z', 'name': '21', 'shape': 'd'}},
{'immediate': {'value': '90'}},
"instruction": "fcmla",
"operands": [
{"register": {"prefix": "z", "name": "26", "shape": "d"}},
{"register": {"prefix": "p", "name": "0", "predication": "m"}},
{"register": {"prefix": "z", "name": "29", "shape": "d"}},
{"register": {"prefix": "z", "name": "21", "shape": "d"}},
{"immediate": {"value": "90"}},
],
'directive': None,
'comment': None,
'label': None,
'line': 'fcmla z26.d, p0/m, z29.d, z21.d, #90',
'line_number': 8,
"directive": None,
"comment": None,
"label": None,
"line": "fcmla z26.d, p0/m, z29.d, z21.d, #90",
"line_number": 8,
}
parsed_1 = self.parser.parse_line(line_comment, 1)
@@ -307,15 +303,15 @@ class TestParserAArch64(unittest.TestCase):
self.assertEqual(len(parsed), 645)
def test_normalize_imd(self):
imd_decimal_1 = {'value': '79'}
imd_hex_1 = {'value': '0x4f'}
imd_decimal_2 = {'value': '8'}
imd_hex_2 = {'value': '0x8'}
imd_float_11 = {'float': {'mantissa': '0.79', 'e_sign': '+', 'exponent': '2'}}
imd_float_12 = {'float': {'mantissa': '790.0', 'e_sign': '-', 'exponent': '1'}}
imd_double_11 = {'double': {'mantissa': '0.79', 'e_sign': '+', 'exponent': '2'}}
imd_double_12 = {'double': {'mantissa': '790.0', 'e_sign': '-', 'exponent': '1'}}
identifier = {'identifier': {'name': '..B1.4'}}
imd_decimal_1 = {"value": "79"}
imd_hex_1 = {"value": "0x4f"}
imd_decimal_2 = {"value": "8"}
imd_hex_2 = {"value": "0x8"}
imd_float_11 = {"float": {"mantissa": "0.79", "e_sign": "+", "exponent": "2"}}
imd_float_12 = {"float": {"mantissa": "790.0", "e_sign": "-", "exponent": "1"}}
imd_double_11 = {"double": {"mantissa": "0.79", "e_sign": "+", "exponent": "2"}}
imd_double_12 = {"double": {"mantissa": "790.0", "e_sign": "-", "exponent": "1"}}
identifier = {"identifier": {"name": "..B1.4"}}
value1 = self.parser.normalize_imd(imd_decimal_1)
self.assertEqual(value1, self.parser.normalize_imd(imd_hex_1))
@@ -329,27 +325,28 @@ class TestParserAArch64(unittest.TestCase):
self.assertEqual(self.parser.normalize_imd(identifier), identifier)
def test_multiple_regs(self):
instr_range = 'PUSH {x5-x7}'
reg_range = AttrDict({
'register': {
'range': [
{'prefix': 'x', 'name': '5'},
{'prefix': 'x', 'name': '7'}
],
'index': None
instr_range = "PUSH {x5-x7}"
reg_range = AttrDict(
{
"register": {
"range": [{"prefix": "x", "name": "5"}, {"prefix": "x", "name": "7"}],
"index": None,
}
})
instr_list = 'POP {x5, x7, x9}'
reg_list = AttrDict({
'register': {
'list': [
{'prefix': 'x', 'name': '5'},
{'prefix': 'x', 'name': '7'},
{'prefix': 'x', 'name': '9'}
],
'index': None
}
})
)
instr_list = "POP {x5, x7, x9}"
reg_list = AttrDict(
{
"register": {
"list": [
{"prefix": "x", "name": "5"},
{"prefix": "x", "name": "7"},
{"prefix": "x", "name": "9"},
],
"index": None,
}
}
)
prange = self.parser.parse_line(instr_range)
plist = self.parser.parse_line(instr_list)
@@ -357,22 +354,22 @@ class TestParserAArch64(unittest.TestCase):
self.assertEqual(plist.operands[0], reg_list)
def test_reg_dependency(self):
reg_1_1 = AttrDict({'prefix': 'b', 'name': '1'})
reg_1_2 = AttrDict({'prefix': 'h', 'name': '1'})
reg_1_3 = AttrDict({'prefix': 's', 'name': '1'})
reg_1_4 = AttrDict({'prefix': 'd', 'name': '1'})
reg_1_4 = AttrDict({'prefix': 'q', 'name': '1'})
reg_2_1 = AttrDict({'prefix': 'w', 'name': '2'})
reg_2_2 = AttrDict({'prefix': 'x', 'name': '2'})
reg_v1_1 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '16', 'shape': 'b'})
reg_v1_2 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '8', 'shape': 'h'})
reg_v1_3 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '4', 'shape': 's'})
reg_v1_4 = AttrDict({'prefix': 'v', 'name': '11', 'lanes': '2', 'shape': 'd'})
reg_1_1 = AttrDict({"prefix": "b", "name": "1"})
reg_1_2 = AttrDict({"prefix": "h", "name": "1"})
reg_1_3 = AttrDict({"prefix": "s", "name": "1"})
reg_1_4 = AttrDict({"prefix": "d", "name": "1"})
reg_1_4 = AttrDict({"prefix": "q", "name": "1"})
reg_2_1 = AttrDict({"prefix": "w", "name": "2"})
reg_2_2 = AttrDict({"prefix": "x", "name": "2"})
reg_v1_1 = AttrDict({"prefix": "v", "name": "11", "lanes": "16", "shape": "b"})
reg_v1_2 = AttrDict({"prefix": "v", "name": "11", "lanes": "8", "shape": "h"})
reg_v1_3 = AttrDict({"prefix": "v", "name": "11", "lanes": "4", "shape": "s"})
reg_v1_4 = AttrDict({"prefix": "v", "name": "11", "lanes": "2", "shape": "d"})
reg_b5 = AttrDict({'prefix': 'b', 'name': '5'})
reg_q15 = AttrDict({'prefix': 'q', 'name': '15'})
reg_v10 = AttrDict({'prefix': 'v', 'name': '10', 'lanes': '2', 'shape': 's'})
reg_v20 = AttrDict({'prefix': 'v', 'name': '20', 'lanes': '2', 'shape': 'd'})
reg_b5 = AttrDict({"prefix": "b", "name": "5"})
reg_q15 = AttrDict({"prefix": "q", "name": "15"})
reg_v10 = AttrDict({"prefix": "v", "name": "10", "lanes": "2", "shape": "s"})
reg_v20 = AttrDict({"prefix": "v", "name": "20", "lanes": "2", "shape": "d"})
reg_1 = [reg_1_1, reg_1_2, reg_1_3, reg_1_4]
reg_2 = [reg_2_1, reg_2_2]
@@ -406,7 +403,7 @@ class TestParserAArch64(unittest.TestCase):
# Helper functions
##################
def _get_comment(self, parser, comment):
return ' '.join(
return " ".join(
AttrDict.convert_dict(
parser.process_operand(parser.comment.parseString(comment, parseAll=True).asDict())
).comment
@@ -425,11 +422,11 @@ class TestParserAArch64(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestParserAArch64)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -15,7 +15,7 @@ class TestParserX86ATT(unittest.TestCase):
@classmethod
def setUpClass(self):
self.parser = ParserX86ATT()
with open(self._find_file('triad_x86_iaca.s')) as f:
with open(self._find_file("triad_x86_iaca.s")) as f:
self.triad_code = f.read()
##################
@@ -23,29 +23,29 @@ class TestParserX86ATT(unittest.TestCase):
##################
def test_comment_parser(self):
self.assertEqual(self._get_comment(self.parser, '# some comments'), 'some comments')
self.assertEqual(self._get_comment(self.parser, '\t\t#AA BB CC \t end \t'), 'AA BB CC end')
self.assertEqual(self._get_comment(self.parser, "# some comments"), "some comments")
self.assertEqual(self._get_comment(self.parser, "\t\t#AA BB CC \t end \t"), "AA BB CC end")
self.assertEqual(
self._get_comment(self.parser, '\t## comment ## comment'), '# comment ## comment'
self._get_comment(self.parser, "\t## comment ## comment"), "# comment ## comment"
)
def test_label_parser(self):
self.assertEqual(self._get_label(self.parser, 'main:').name, 'main')
self.assertEqual(self._get_label(self.parser, '..B1.10:').name, '..B1.10')
self.assertEqual(self._get_label(self.parser, '.2.3_2_pack.3:').name, '.2.3_2_pack.3')
self.assertEqual(self._get_label(self.parser, '.L1:\t\t\t#label1').name, '.L1')
self.assertEqual(self._get_label(self.parser, "main:").name, "main")
self.assertEqual(self._get_label(self.parser, "..B1.10:").name, "..B1.10")
self.assertEqual(self._get_label(self.parser, ".2.3_2_pack.3:").name, ".2.3_2_pack.3")
self.assertEqual(self._get_label(self.parser, ".L1:\t\t\t#label1").name, ".L1")
self.assertEqual(
' '.join(self._get_label(self.parser, '.L1:\t\t\t#label1').comment), 'label1'
" ".join(self._get_label(self.parser, ".L1:\t\t\t#label1").comment), "label1"
)
with self.assertRaises(ParseException):
self._get_label(self.parser, '\t.cfi_startproc')
self._get_label(self.parser, "\t.cfi_startproc")
def test_directive_parser(self):
self.assertEqual(self._get_directive(self.parser, '\t.text').name, 'text')
self.assertEqual(len(self._get_directive(self.parser, '\t.text').parameters), 0)
self.assertEqual(self._get_directive(self.parser, '\t.align\t16,0x90').name, 'align')
self.assertEqual(len(self._get_directive(self.parser, '\t.align\t16,0x90').parameters), 2)
self.assertEqual(len(self._get_directive(self.parser, '.text').parameters), 0)
self.assertEqual(self._get_directive(self.parser, "\t.text").name, "text")
self.assertEqual(len(self._get_directive(self.parser, "\t.text").parameters), 0)
self.assertEqual(self._get_directive(self.parser, "\t.align\t16,0x90").name, "align")
self.assertEqual(len(self._get_directive(self.parser, "\t.align\t16,0x90").parameters), 2)
self.assertEqual(len(self._get_directive(self.parser, ".text").parameters), 0)
self.assertEqual(
len(self._get_directive(self.parser, '.file\t1 "path/to/file.c"').parameters), 2
)
@@ -54,54 +54,52 @@ class TestParserX86ATT(unittest.TestCase):
'"path/to/file.c"',
)
self.assertEqual(
self._get_directive(self.parser, '\t.set\tL$set$0,LECIE1-LSCIE1').parameters,
['L$set$0', 'LECIE1-LSCIE1'],
self._get_directive(self.parser, "\t.set\tL$set$0,LECIE1-LSCIE1").parameters,
["L$set$0", "LECIE1-LSCIE1"],
)
self.assertEqual(
self._get_directive(
self.parser,
'\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support',
"\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support",
).parameters,
['__TEXT', '__eh_frame', 'coalesced', 'no_toc+strip_static_syms+live_support'],
["__TEXT", "__eh_frame", "coalesced", "no_toc+strip_static_syms+live_support"],
)
self.assertEqual(
self._get_directive(
self.parser, '\t.section\t__TEXT,__literal16,16byte_literals'
self.parser, "\t.section\t__TEXT,__literal16,16byte_literals"
).parameters,
['__TEXT', '__literal16', '16byte_literals'],
["__TEXT", "__literal16", "16byte_literals"],
)
self.assertEqual(
self._get_directive(self.parser, '\t.align\t16,0x90').parameters[1], '0x90'
self._get_directive(self.parser, "\t.align\t16,0x90").parameters[1], "0x90"
)
self.assertEqual(
self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[
'name'
],
'byte',
self._get_directive(self.parser, " .byte 100,103,144 #IACA START")["name"],
"byte",
)
self.assertEqual(
self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[
'parameters'
self._get_directive(self.parser, " .byte 100,103,144 #IACA START")[
"parameters"
][2],
'144',
"144",
)
self.assertEqual(
' '.join(
self._get_directive(self.parser, ' .byte 100,103,144 #IACA START')[
'comment'
" ".join(
self._get_directive(self.parser, " .byte 100,103,144 #IACA START")[
"comment"
]
),
'IACA START',
"IACA START",
)
def test_parse_instruction(self):
instr1 = '\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27'
instr2 = 'jb ..B1.4 \t'
instr3 = ' movl $222,%ebx #IACA END'
instr4 = 'vmovss %xmm4, -4(%rsp,%rax,8) #12.9'
instr5 = 'mov %ebx,var(,1)'
instr6 = 'lea (,%rax,8),%rbx'
instr7 = 'vinsertf128 $0x1, %xmm0, %ymm1, %ymm1'
instr1 = "\t\tvcvtsi2ss %edx, %xmm2, %xmm2\t\t\t#12.27"
instr2 = "jb ..B1.4 \t"
instr3 = " movl $222,%ebx #IACA END"
instr4 = "vmovss %xmm4, -4(%rsp,%rax,8) #12.9"
instr5 = "mov %ebx,var(,1)"
instr6 = "lea (,%rax,8),%rbx"
instr7 = "vinsertf128 $0x1, %xmm0, %ymm1, %ymm1"
parsed_1 = self.parser.parse_instruction(instr1)
parsed_2 = self.parser.parse_instruction(instr2)
@@ -111,99 +109,99 @@ class TestParserX86ATT(unittest.TestCase):
parsed_6 = self.parser.parse_instruction(instr6)
parsed_7 = self.parser.parse_instruction(instr7)
self.assertEqual(parsed_1.instruction, 'vcvtsi2ss')
self.assertEqual(parsed_1.operands[0].register.name, 'edx')
self.assertEqual(parsed_1.operands[1].register.name, 'xmm2')
self.assertEqual(parsed_1.comment, '12.27')
self.assertEqual(parsed_1.instruction, "vcvtsi2ss")
self.assertEqual(parsed_1.operands[0].register.name, "edx")
self.assertEqual(parsed_1.operands[1].register.name, "xmm2")
self.assertEqual(parsed_1.comment, "12.27")
self.assertEqual(parsed_2.instruction, 'jb')
self.assertEqual(parsed_2.operands[0].identifier.name, '..B1.4')
self.assertEqual(parsed_2.instruction, "jb")
self.assertEqual(parsed_2.operands[0].identifier.name, "..B1.4")
self.assertEqual(len(parsed_2.operands), 1)
self.assertIsNone(parsed_2.comment)
self.assertEqual(parsed_3.instruction, 'movl')
self.assertEqual(parsed_3.operands[0].immediate.value, '222')
self.assertEqual(parsed_3.operands[1].register.name, 'ebx')
self.assertEqual(parsed_3.comment, 'IACA END')
self.assertEqual(parsed_3.instruction, "movl")
self.assertEqual(parsed_3.operands[0].immediate.value, "222")
self.assertEqual(parsed_3.operands[1].register.name, "ebx")
self.assertEqual(parsed_3.comment, "IACA END")
self.assertEqual(parsed_4.instruction, 'vmovss')
self.assertEqual(parsed_4.operands[1].memory.offset.value, '-4')
self.assertEqual(parsed_4.operands[1].memory.base.name, 'rsp')
self.assertEqual(parsed_4.operands[1].memory.index.name, 'rax')
self.assertEqual(parsed_4.instruction, "vmovss")
self.assertEqual(parsed_4.operands[1].memory.offset.value, "-4")
self.assertEqual(parsed_4.operands[1].memory.base.name, "rsp")
self.assertEqual(parsed_4.operands[1].memory.index.name, "rax")
self.assertEqual(parsed_4.operands[1].memory.scale, 8)
self.assertEqual(parsed_4.operands[0].register.name, 'xmm4')
self.assertEqual(parsed_4.comment, '12.9')
self.assertEqual(parsed_4.operands[0].register.name, "xmm4")
self.assertEqual(parsed_4.comment, "12.9")
self.assertEqual(parsed_5.instruction, 'mov')
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, 'var')
self.assertEqual(parsed_5.instruction, "mov")
self.assertEqual(parsed_5.operands[1].memory.offset.identifier.name, "var")
self.assertIsNone(parsed_5.operands[1].memory.base)
self.assertIsNone(parsed_5.operands[1].memory.index)
self.assertEqual(parsed_5.operands[1].memory.scale, 1)
self.assertEqual(parsed_5.operands[0].register.name, 'ebx')
self.assertEqual(parsed_5.operands[0].register.name, "ebx")
self.assertEqual(parsed_6.instruction, 'lea')
self.assertEqual(parsed_6.instruction, "lea")
self.assertIsNone(parsed_6.operands[0].memory.offset)
self.assertIsNone(parsed_6.operands[0].memory.base)
self.assertEqual(parsed_6.operands[0].memory.index.name, 'rax')
self.assertEqual(parsed_6.operands[0].memory.index.name, "rax")
self.assertEqual(parsed_6.operands[0].memory.scale, 8)
self.assertEqual(parsed_6.operands[1].register.name, 'rbx')
self.assertEqual(parsed_6.operands[1].register.name, "rbx")
self.assertEqual(parsed_7.operands[0].immediate.value, '0x1')
self.assertEqual(parsed_7.operands[1].register.name, 'xmm0')
self.assertEqual(parsed_7.operands[2].register.name, 'ymm1')
self.assertEqual(parsed_7.operands[3].register.name, 'ymm1')
self.assertEqual(parsed_7.operands[0].immediate.value, "0x1")
self.assertEqual(parsed_7.operands[1].register.name, "xmm0")
self.assertEqual(parsed_7.operands[2].register.name, "ymm1")
self.assertEqual(parsed_7.operands[3].register.name, "ymm1")
def test_parse_line(self):
line_comment = '# -- Begin main'
line_label = '..B1.7: # Preds ..B1.6'
line_directive = '.quad .2.3_2__kmpc_loc_pack.2 #qed'
line_instruction = 'lea 2(%rax,%rax), %ecx #12.9'
line_comment = "# -- Begin main"
line_label = "..B1.7: # Preds ..B1.6"
line_directive = ".quad .2.3_2__kmpc_loc_pack.2 #qed"
line_instruction = "lea 2(%rax,%rax), %ecx #12.9"
instruction_form_1 = {
'instruction': None,
'operands': [],
'directive': None,
'comment': '-- Begin main',
'label': None,
'line': '# -- Begin main',
'line_number': 1,
"instruction": None,
"operands": [],
"directive": None,
"comment": "-- Begin main",
"label": None,
"line": "# -- Begin main",
"line_number": 1,
}
instruction_form_2 = {
'instruction': None,
'operands': [],
'directive': None,
'comment': 'Preds ..B1.6',
'label': '..B1.7',
'line': '..B1.7: # Preds ..B1.6',
'line_number': 2,
"instruction": None,
"operands": [],
"directive": None,
"comment": "Preds ..B1.6",
"label": "..B1.7",
"line": "..B1.7: # Preds ..B1.6",
"line_number": 2,
}
instruction_form_3 = {
'instruction': None,
'operands': [],
'directive': {'name': 'quad', 'parameters': ['.2.3_2__kmpc_loc_pack.2']},
'comment': 'qed',
'label': None,
'line': '.quad .2.3_2__kmpc_loc_pack.2 #qed',
'line_number': 3,
"instruction": None,
"operands": [],
"directive": {"name": "quad", "parameters": [".2.3_2__kmpc_loc_pack.2"]},
"comment": "qed",
"label": None,
"line": ".quad .2.3_2__kmpc_loc_pack.2 #qed",
"line_number": 3,
}
instruction_form_4 = {
'instruction': 'lea',
'operands': [
"instruction": "lea",
"operands": [
{
'memory': {
'offset': {'value': '2'},
'base': {'name': 'rax'},
'index': {'name': 'rax'},
'scale': 1,
"memory": {
"offset": {"value": "2"},
"base": {"name": "rax"},
"index": {"name": "rax"},
"scale": 1,
}
},
{'register': {'name': 'ecx'}},
{"register": {"name": "ecx"}},
],
'directive': None,
'comment': '12.9',
'label': None,
'line': 'lea 2(%rax,%rax), %ecx #12.9',
'line_number': 4,
"directive": None,
"comment": "12.9",
"label": None,
"line": "lea 2(%rax,%rax), %ecx #12.9",
"line_number": 4,
}
parsed_1 = self.parser.parse_line(line_comment, 1)
@@ -222,27 +220,27 @@ class TestParserX86ATT(unittest.TestCase):
self.assertEqual(len(parsed), 353)
def test_parse_register(self):
register_str_1 = '%rax'
register_str_2 = '%r9'
register_str_3 = '%xmm1'
register_str_4 = '%rip'
register_str_1 = "%rax"
register_str_2 = "%r9"
register_str_3 = "%xmm1"
register_str_4 = "%rip"
parsed_reg_1 = {'register': {'name': 'rax'}}
parsed_reg_2 = {'register': {'name': 'r9'}}
parsed_reg_3 = {'register': {'name': 'xmm1'}}
parsed_reg_4 = {'register': {'name': 'rip'}}
parsed_reg_1 = {"register": {"name": "rax"}}
parsed_reg_2 = {"register": {"name": "r9"}}
parsed_reg_3 = {"register": {"name": "xmm1"}}
parsed_reg_4 = {"register": {"name": "rip"}}
self.assertEqual(self.parser.parse_register(register_str_1), parsed_reg_1)
self.assertEqual(self.parser.parse_register(register_str_2), parsed_reg_2)
self.assertEqual(self.parser.parse_register(register_str_3), parsed_reg_3)
self.assertEqual(self.parser.parse_register(register_str_4), parsed_reg_4)
self.assertIsNone(self.parser.parse_register('rax'))
self.assertIsNone(self.parser.parse_register("rax"))
def test_normalize_imd(self):
imd_decimal_1 = {'value': '79'}
imd_hex_1 = {'value': '0x4f'}
imd_decimal_2 = {'value': '8'}
imd_hex_2 = {'value': '0x8'}
imd_decimal_1 = {"value": "79"}
imd_hex_1 = {"value": "0x4f"}
imd_decimal_2 = {"value": "8"}
imd_hex_2 = {"value": "0x8"}
self.assertEqual(
self.parser.normalize_imd(imd_decimal_1), self.parser.normalize_imd(imd_hex_1)
)
@@ -251,22 +249,22 @@ class TestParserX86ATT(unittest.TestCase):
)
def test_reg_dependency(self):
reg_a1 = AttrDict({'name': 'rax'})
reg_a2 = AttrDict({'name': 'eax'})
reg_a3 = AttrDict({'name': 'ax'})
reg_a4 = AttrDict({'name': 'al'})
reg_r11 = AttrDict({'name': 'r11'})
reg_r11b = AttrDict({'name': 'r11b'})
reg_r11d = AttrDict({'name': 'r11d'})
reg_r11w = AttrDict({'name': 'r11w'})
reg_xmm1 = AttrDict({'name': 'xmm1'})
reg_ymm1 = AttrDict({'name': 'ymm1'})
reg_zmm1 = AttrDict({'name': 'zmm1'})
reg_a1 = AttrDict({"name": "rax"})
reg_a2 = AttrDict({"name": "eax"})
reg_a3 = AttrDict({"name": "ax"})
reg_a4 = AttrDict({"name": "al"})
reg_r11 = AttrDict({"name": "r11"})
reg_r11b = AttrDict({"name": "r11b"})
reg_r11d = AttrDict({"name": "r11d"})
reg_r11w = AttrDict({"name": "r11w"})
reg_xmm1 = AttrDict({"name": "xmm1"})
reg_ymm1 = AttrDict({"name": "ymm1"})
reg_zmm1 = AttrDict({"name": "zmm1"})
reg_b1 = AttrDict({'name': 'rbx'})
reg_r15 = AttrDict({'name': 'r15'})
reg_xmm2 = AttrDict({'name': 'xmm2'})
reg_ymm3 = AttrDict({'name': 'ymm3'})
reg_b1 = AttrDict({"name": "rbx"})
reg_r15 = AttrDict({"name": "r15"})
reg_xmm2 = AttrDict({"name": "xmm2"})
reg_ymm3 = AttrDict({"name": "ymm3"})
reg_a = [reg_a1, reg_a2, reg_a3, reg_a4]
reg_r = [reg_r11, reg_r11b, reg_r11d, reg_r11w]
@@ -300,7 +298,7 @@ class TestParserX86ATT(unittest.TestCase):
# Helper functions
##################
def _get_comment(self, parser, comment):
return ' '.join(
return " ".join(
AttrDict.convert_dict(
parser.process_operand(parser.comment.parseString(comment, parseAll=True).asDict())
).comment
@@ -319,11 +317,11 @@ class TestParserX86ATT(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestParserX86ATT)
unittest.TextTestRunner(verbosity=2).run(suite)

View File

@@ -12,13 +12,12 @@ import networkx as nx
from osaca.osaca import get_unmatched_instruction_ratio
from osaca.parser import AttrDict, ParserAArch64, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG,
MachineModel, reduce_to_section)
from osaca.semantics import INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section
class TestSemanticTools(unittest.TestCase):
MODULE_DATA_DIR = os.path.join(
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/'
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), "osaca/data/"
)
@classmethod
@@ -26,30 +25,30 @@ class TestSemanticTools(unittest.TestCase):
# set up parser and kernels
cls.parser_x86 = ParserX86ATT()
cls.parser_AArch64 = ParserAArch64()
with open(cls._find_file('kernel_x86.s')) as f:
with open(cls._find_file("kernel_x86.s")) as f:
cls.code_x86 = f.read()
with open(cls._find_file('kernel_aarch64.s')) as f:
with open(cls._find_file("kernel_aarch64.s")) as f:
cls.code_AArch64 = f.read()
cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), 'x86')
cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), "x86")
cls.kernel_AArch64 = reduce_to_section(
cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64'
cls.parser_AArch64.parse_file(cls.code_AArch64), "aarch64"
)
# set up machine models
cls.machine_model_csx = MachineModel(
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml')
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "csx.yml")
)
cls.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml')
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "tx2.yml")
)
cls.semantics_csx = ArchSemantics(
cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/x86.yml')
cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/x86.yml")
)
cls.semantics_tx2 = ArchSemantics(
cls.machine_model_tx2,
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'),
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/aarch64.yml"),
)
cls.machine_model_zen = MachineModel(arch='zen1')
cls.machine_model_zen = MachineModel(arch="zen1")
for i in range(len(cls.kernel_x86)):
cls.semantics_csx.assign_src_dst(cls.kernel_x86[i])
@@ -64,7 +63,7 @@ class TestSemanticTools(unittest.TestCase):
def test_creation_by_name(self):
try:
tmp_mm = MachineModel(arch='CSX')
tmp_mm = MachineModel(arch="CSX")
ArchSemantics(tmp_mm)
except ValueError:
self.fail()
@@ -72,12 +71,12 @@ class TestSemanticTools(unittest.TestCase):
def test_machine_model_various_functions(self):
# check dummy MachineModel creation
try:
MachineModel(isa='x86')
MachineModel(isa='aarch64')
MachineModel(isa="x86")
MachineModel(isa="aarch64")
except ValueError:
self.fail()
test_mm_x86 = MachineModel(path_to_yaml=self._find_file('test_db_x86.yml'))
test_mm_arm = MachineModel(path_to_yaml=self._find_file('test_db_aarch64.yml'))
test_mm_x86 = MachineModel(path_to_yaml=self._find_file("test_db_x86.yml"))
test_mm_arm = MachineModel(path_to_yaml=self._find_file("test_db_aarch64.yml"))
# test get_instruction without mnemonic
self.assertIsNone(test_mm_x86.get_instruction(None, []))
@@ -86,80 +85,80 @@ class TestSemanticTools(unittest.TestCase):
# test get_instruction from DB
self.assertIsNone(test_mm_x86.get_instruction(None, []))
self.assertIsNone(test_mm_arm.get_instruction(None, []))
self.assertIsNone(test_mm_x86.get_instruction('NOT_IN_DB', []))
self.assertIsNone(test_mm_arm.get_instruction('NOT_IN_DB', []))
name_x86_1 = 'vaddpd'
self.assertIsNone(test_mm_x86.get_instruction("NOT_IN_DB", []))
self.assertIsNone(test_mm_arm.get_instruction("NOT_IN_DB", []))
name_x86_1 = "vaddpd"
operands_x86_1 = [
{'class': 'register', 'name': 'xmm'},
{'class': 'register', 'name': 'xmm'},
{'class': 'register', 'name': 'xmm'},
{"class": "register", "name": "xmm"},
{"class": "register", "name": "xmm"},
{"class": "register", "name": "xmm"},
]
instr_form_x86_1 = test_mm_x86.get_instruction(name_x86_1, operands_x86_1)
self.assertEqual(instr_form_x86_1, test_mm_x86.get_instruction(name_x86_1, operands_x86_1))
self.assertEqual(
test_mm_x86.get_instruction('jg', [{'class': 'identifier'}]),
test_mm_x86.get_instruction('jg', [{'class': 'identifier'}]),
test_mm_x86.get_instruction("jg", [{"class": "identifier"}]),
test_mm_x86.get_instruction("jg", [{"class": "identifier"}]),
)
name_arm_1 = 'fadd'
name_arm_1 = "fadd"
operands_arm_1 = [
{'class': 'register', 'prefix': 'v', 'shape': 's'},
{'class': 'register', 'prefix': 'v', 'shape': 's'},
{'class': 'register', 'prefix': 'v', 'shape': 's'},
{"class": "register", "prefix": "v", "shape": "s"},
{"class": "register", "prefix": "v", "shape": "s"},
{"class": "register", "prefix": "v", "shape": "s"},
]
instr_form_arm_1 = test_mm_arm.get_instruction(name_arm_1, operands_arm_1)
self.assertEqual(instr_form_arm_1, test_mm_arm.get_instruction(name_arm_1, operands_arm_1))
self.assertEqual(
test_mm_arm.get_instruction('b.ne', [{'class': 'identifier'}]),
test_mm_arm.get_instruction('b.ne', [{'class': 'identifier'}]),
test_mm_arm.get_instruction("b.ne", [{"class": "identifier"}]),
test_mm_arm.get_instruction("b.ne", [{"class": "identifier"}]),
)
# test full instruction name
self.assertEqual(
MachineModel.get_full_instruction_name(instr_form_x86_1),
'vaddpd register(name:xmm),register(name:xmm),register(name:xmm)',
"vaddpd register(name:xmm),register(name:xmm),register(name:xmm)",
)
self.assertEqual(
MachineModel.get_full_instruction_name(instr_form_arm_1),
'fadd register(prefix:v,shape:s),register(prefix:v,shape:s),'
+ 'register(prefix:v,shape:s)',
"fadd register(prefix:v,shape:s),register(prefix:v,shape:s),"
+ "register(prefix:v,shape:s)",
)
# test get_store_tp
self.assertEqual(
test_mm_x86.get_store_throughput(
{'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': 1}
{"base": {"name": "x"}, "offset": None, "index": None, "scale": 1}
),
[[2, '237'], [2, '4']],
[[2, "237"], [2, "4"]],
)
self.assertEqual(
test_mm_x86.get_store_throughput(
{'base': {'prefix': 'NOT_IN_DB'}, 'offset': None, 'index': 'NOT_NONE', 'scale': 1}
{"base": {"prefix": "NOT_IN_DB"}, "offset": None, "index": "NOT_NONE", "scale": 1}
),
[[1, '23'], [1, '4']],
[[1, "23"], [1, "4"]],
)
self.assertEqual(
test_mm_arm.get_store_throughput(
{'base': {'prefix': 'x'}, 'offset': None, 'index': None, 'scale': 1}
{"base": {"prefix": "x"}, "offset": None, "index": None, "scale": 1}
),
[[2, '34'], [2, '5']],
[[2, "34"], [2, "5"]],
)
self.assertEqual(
test_mm_arm.get_store_throughput(
{'base': {'prefix': 'NOT_IN_DB'}, 'offset': None, 'index': None, 'scale': 1}
{"base": {"prefix": "NOT_IN_DB"}, "offset": None, "index": None, "scale": 1}
),
[[1, '34'], [1, '5']],
[[1, "34"], [1, "5"]],
)
# test get_store_lt
self.assertEqual(
test_mm_x86.get_store_latency(
{'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': '1'}
{"base": {"name": "x"}, "offset": None, "index": None, "scale": "1"}
),
0,
)
self.assertEqual(
test_mm_arm.get_store_latency(
{'base': {'prefix': 'x'}, 'offset': None, 'index': None, 'scale': '1'}
{"base": {"prefix": "x"}, "offset": None, "index": None, "scale": "1"}
),
0,
)
@@ -170,55 +169,55 @@ class TestSemanticTools(unittest.TestCase):
# test default load tp
self.assertEqual(
test_mm_x86.get_load_throughput(
{'base': {'name': 'x'}, 'offset': None, 'index': None, 'scale': 1}
{"base": {"name": "x"}, "offset": None, "index": None, "scale": 1}
),
[[1, '23'], [1, ['2D', '3D']]],
[[1, "23"], [1, ["2D", "3D"]]],
)
# test adding port
test_mm_x86.add_port('dummyPort')
test_mm_arm.add_port('dummyPort')
test_mm_x86.add_port("dummyPort")
test_mm_arm.add_port("dummyPort")
# test dump of DB
with open('/dev/null', 'w') as dev_null:
with open("/dev/null", "w") as dev_null:
test_mm_x86.dump(stream=dev_null)
test_mm_arm.dump(stream=dev_null)
def test_src_dst_assignment_x86(self):
for instruction_form in self.kernel_x86:
with self.subTest(instruction_form=instruction_form):
if instruction_form['semantic_operands'] is not None:
self.assertTrue('source' in instruction_form['semantic_operands'])
self.assertTrue('destination' in instruction_form['semantic_operands'])
self.assertTrue('src_dst' in instruction_form['semantic_operands'])
if instruction_form["semantic_operands"] is not None:
self.assertTrue("source" in instruction_form["semantic_operands"])
self.assertTrue("destination" in instruction_form["semantic_operands"])
self.assertTrue("src_dst" in instruction_form["semantic_operands"])
def test_src_dst_assignment_AArch64(self):
for instruction_form in self.kernel_AArch64:
with self.subTest(instruction_form=instruction_form):
if instruction_form['semantic_operands'] is not None:
self.assertTrue('source' in instruction_form['semantic_operands'])
self.assertTrue('destination' in instruction_form['semantic_operands'])
self.assertTrue('src_dst' in instruction_form['semantic_operands'])
if instruction_form["semantic_operands"] is not None:
self.assertTrue("source" in instruction_form["semantic_operands"])
self.assertTrue("destination" in instruction_form["semantic_operands"])
self.assertTrue("src_dst" in instruction_form["semantic_operands"])
def test_tp_lt_assignment_x86(self):
self.assertTrue('ports' in self.machine_model_csx)
port_num = len(self.machine_model_csx['ports'])
self.assertTrue("ports" in self.machine_model_csx)
port_num = len(self.machine_model_csx["ports"])
for instruction_form in self.kernel_x86:
with self.subTest(instruction_form=instruction_form):
self.assertTrue('throughput' in instruction_form)
self.assertTrue('latency' in instruction_form)
self.assertIsInstance(instruction_form['port_pressure'], list)
self.assertEqual(len(instruction_form['port_pressure']), port_num)
self.assertTrue("throughput" in instruction_form)
self.assertTrue("latency" in instruction_form)
self.assertIsInstance(instruction_form["port_pressure"], list)
self.assertEqual(len(instruction_form["port_pressure"]), port_num)
def test_tp_lt_assignment_AArch64(self):
self.assertTrue('ports' in self.machine_model_tx2)
port_num = len(self.machine_model_tx2['ports'])
self.assertTrue("ports" in self.machine_model_tx2)
port_num = len(self.machine_model_tx2["ports"])
for instruction_form in self.kernel_AArch64:
with self.subTest(instruction_form=instruction_form):
self.assertTrue('throughput' in instruction_form)
self.assertTrue('latency' in instruction_form)
self.assertIsInstance(instruction_form['port_pressure'], list)
self.assertEqual(len(instruction_form['port_pressure']), port_num)
self.assertTrue("throughput" in instruction_form)
self.assertTrue("latency" in instruction_form)
self.assertIsInstance(instruction_form["port_pressure"], list)
self.assertEqual(len(instruction_form["port_pressure"]), port_num)
def test_optimal_throughput_assignment(self):
# x86
@@ -266,7 +265,7 @@ class TestSemanticTools(unittest.TestCase):
with self.assertRaises(ValueError):
dg.get_dependent_instruction_forms()
# test dot creation
dg.export_graph(filepath='/dev/null')
dg.export_graph(filepath="/dev/null")
def test_kernelDG_AArch64(self):
dg = KernelDG(self.kernel_AArch64, self.parser_AArch64, self.machine_model_tx2)
@@ -292,11 +291,11 @@ class TestSemanticTools(unittest.TestCase):
with self.assertRaises(ValueError):
dg.get_dependent_instruction_forms()
# test dot creation
dg.export_graph(filepath='/dev/null')
dg.export_graph(filepath="/dev/null")
def test_hidden_load(self):
machine_model_hld = MachineModel(
path_to_yaml=self._find_file('hidden_load_machine_model.yml')
path_to_yaml=self._find_file("hidden_load_machine_model.yml")
)
self.assertTrue(machine_model_hld.has_hidden_loads())
semantics_hld = ArchSemantics(machine_model_hld)
@@ -308,9 +307,9 @@ class TestSemanticTools(unittest.TestCase):
semantics_hld.add_semantics(kernel_hld_2)
semantics_hld.add_semantics(kernel_hld_3)
num_hidden_loads = len([x for x in kernel_hld if INSTR_FLAGS.HIDDEN_LD in x['flags']])
num_hidden_loads_2 = len([x for x in kernel_hld_2 if INSTR_FLAGS.HIDDEN_LD in x['flags']])
num_hidden_loads_3 = len([x for x in kernel_hld_3 if INSTR_FLAGS.HIDDEN_LD in x['flags']])
num_hidden_loads = len([x for x in kernel_hld if INSTR_FLAGS.HIDDEN_LD in x["flags"]])
num_hidden_loads_2 = len([x for x in kernel_hld_2 if INSTR_FLAGS.HIDDEN_LD in x["flags"]])
num_hidden_loads_3 = len([x for x in kernel_hld_3 if INSTR_FLAGS.HIDDEN_LD in x["flags"]])
self.assertEqual(num_hidden_loads, 1)
self.assertEqual(num_hidden_loads_2, 0)
self.assertEqual(num_hidden_loads_3, 1)
@@ -333,42 +332,42 @@ class TestSemanticTools(unittest.TestCase):
self.assertEqual(len(lc_deps), 2)
# ID 8
self.assertEqual(
lc_deps[lcd_id]['root'], dg.dg.nodes(data=True)[lcd_id]['instruction_form']
lc_deps[lcd_id]["root"], dg.dg.nodes(data=True)[lcd_id]["instruction_form"]
)
self.assertEqual(len(lc_deps[lcd_id]['dependencies']), 1)
self.assertEqual(len(lc_deps[lcd_id]["dependencies"]), 1)
self.assertEqual(
lc_deps[lcd_id]['dependencies'][0], dg.dg.nodes(data=True)[lcd_id]['instruction_form']
lc_deps[lcd_id]["dependencies"][0], dg.dg.nodes(data=True)[lcd_id]["instruction_form"]
)
# w/ flag dependencies: ID 9 w/ len=2
# w/o flag dependencies: ID 5 w/ len=1
# TODO discuss
self.assertEqual(
lc_deps[lcd_id2]['root'], dg.dg.nodes(data=True)[lcd_id2]['instruction_form']
lc_deps[lcd_id2]["root"], dg.dg.nodes(data=True)[lcd_id2]["instruction_form"]
)
self.assertEqual(len(lc_deps[lcd_id2]['dependencies']), 1)
self.assertEqual(len(lc_deps[lcd_id2]["dependencies"]), 1)
self.assertEqual(
lc_deps[lcd_id2]['dependencies'][0],
dg.dg.nodes(data=True)[lcd_id2]['instruction_form'],
lc_deps[lcd_id2]["dependencies"][0],
dg.dg.nodes(data=True)[lcd_id2]["instruction_form"],
)
def test_is_read_is_written_x86(self):
# independent form HW model
dag = KernelDG(self.kernel_x86, self.parser_x86, None)
reg_rcx = AttrDict({'name': 'rcx'})
reg_ymm1 = AttrDict({'name': 'ymm1'})
reg_rcx = AttrDict({"name": "rcx"})
reg_ymm1 = AttrDict({"name": "ymm1"})
instr_form_r_c = self.parser_x86.parse_line('vmovsd %xmm0, (%r15,%rcx,8)')
instr_form_r_c = self.parser_x86.parse_line("vmovsd %xmm0, (%r15,%rcx,8)")
self.semantics_csx.assign_src_dst(instr_form_r_c)
instr_form_non_r_c = self.parser_x86.parse_line('movl %xmm0, (%r15,%rax,8)')
instr_form_non_r_c = self.parser_x86.parse_line("movl %xmm0, (%r15,%rax,8)")
self.semantics_csx.assign_src_dst(instr_form_non_r_c)
instr_form_w_c = self.parser_x86.parse_line('movi $0x05ACA, %rcx')
instr_form_w_c = self.parser_x86.parse_line("movi $0x05ACA, %rcx")
self.semantics_csx.assign_src_dst(instr_form_w_c)
instr_form_rw_ymm_1 = self.parser_x86.parse_line('vinsertf128 $0x1, %xmm1, %ymm0, %ymm1')
instr_form_rw_ymm_1 = self.parser_x86.parse_line("vinsertf128 $0x1, %xmm1, %ymm0, %ymm1")
self.semantics_csx.assign_src_dst(instr_form_rw_ymm_1)
instr_form_rw_ymm_2 = self.parser_x86.parse_line('vinsertf128 $0x1, %xmm0, %ymm1, %ymm1')
instr_form_rw_ymm_2 = self.parser_x86.parse_line("vinsertf128 $0x1, %xmm0, %ymm1, %ymm1")
self.semantics_csx.assign_src_dst(instr_form_rw_ymm_2)
instr_form_r_ymm = self.parser_x86.parse_line('vmovapd %ymm1, %ymm0')
instr_form_r_ymm = self.parser_x86.parse_line("vmovapd %ymm1, %ymm0")
self.semantics_csx.assign_src_dst(instr_form_r_ymm)
self.assertTrue(dag.is_read(reg_rcx, instr_form_r_c))
@@ -387,29 +386,29 @@ class TestSemanticTools(unittest.TestCase):
def test_is_read_is_written_AArch64(self):
# independent form HW model
dag = KernelDG(self.kernel_AArch64, self.parser_AArch64, None)
reg_x1 = AttrDict({'prefix': 'x', 'name': '1'})
reg_w1 = AttrDict({'prefix': 'w', 'name': '1'})
reg_d1 = AttrDict({'prefix': 'd', 'name': '1'})
reg_q1 = AttrDict({'prefix': 'q', 'name': '1'})
reg_v1 = AttrDict({'prefix': 'v', 'name': '1', 'lanes': '2', 'shape': 'd'})
reg_x1 = AttrDict({"prefix": "x", "name": "1"})
reg_w1 = AttrDict({"prefix": "w", "name": "1"})
reg_d1 = AttrDict({"prefix": "d", "name": "1"})
reg_q1 = AttrDict({"prefix": "q", "name": "1"})
reg_v1 = AttrDict({"prefix": "v", "name": "1", "lanes": "2", "shape": "d"})
regs = [reg_d1, reg_q1, reg_v1]
regs_gp = [reg_w1, reg_x1]
instr_form_r_1 = self.parser_AArch64.parse_line('stp q1, q3, [x12, #192]')
instr_form_r_1 = self.parser_AArch64.parse_line("stp q1, q3, [x12, #192]")
self.semantics_tx2.assign_src_dst(instr_form_r_1)
instr_form_r_2 = self.parser_AArch64.parse_line('fadd v2.2d, v1.2d, v0.2d')
instr_form_r_2 = self.parser_AArch64.parse_line("fadd v2.2d, v1.2d, v0.2d")
self.semantics_tx2.assign_src_dst(instr_form_r_2)
instr_form_w_1 = self.parser_AArch64.parse_line('ldr d1, [x1, #:got_lo12:q2c]')
instr_form_w_1 = self.parser_AArch64.parse_line("ldr d1, [x1, #:got_lo12:q2c]")
self.semantics_tx2.assign_src_dst(instr_form_w_1)
instr_form_non_w_1 = self.parser_AArch64.parse_line('ldr x1, [x1, #:got_lo12:q2c]')
instr_form_non_w_1 = self.parser_AArch64.parse_line("ldr x1, [x1, #:got_lo12:q2c]")
self.semantics_tx2.assign_src_dst(instr_form_non_w_1)
instr_form_rw_1 = self.parser_AArch64.parse_line('fmul v1.2d, v1.2d, v0.2d')
instr_form_rw_1 = self.parser_AArch64.parse_line("fmul v1.2d, v1.2d, v0.2d")
self.semantics_tx2.assign_src_dst(instr_form_rw_1)
instr_form_rw_2 = self.parser_AArch64.parse_line('ldp q2, q4, [x1, #64]!')
instr_form_rw_2 = self.parser_AArch64.parse_line("ldp q2, q4, [x1, #64]!")
self.semantics_tx2.assign_src_dst(instr_form_rw_2)
instr_form_rw_3 = self.parser_AArch64.parse_line('str x4, [x1], #64')
instr_form_rw_3 = self.parser_AArch64.parse_line("str x4, [x1], #64")
self.semantics_tx2.assign_src_dst(instr_form_rw_3)
instr_form_non_rw_1 = self.parser_AArch64.parse_line('adds x1, x11')
instr_form_non_rw_1 = self.parser_AArch64.parse_line("adds x1, x11")
self.semantics_tx2.assign_src_dst(instr_form_non_rw_1)
for reg in regs:
@@ -447,43 +446,43 @@ class TestSemanticTools(unittest.TestCase):
with self.assertRaises(ValueError):
MachineModel()
with self.assertRaises(ValueError):
MachineModel(arch='CSX', path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml'))
MachineModel(arch="CSX", path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "csx.yml"))
with self.assertRaises(FileNotFoundError):
MachineModel(arch='THE_MACHINE')
MachineModel(arch="THE_MACHINE")
with self.assertRaises(FileNotFoundError):
MachineModel(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'THE_MACHINE.yml'))
MachineModel(path_to_yaml=os.path.join(self.MODULE_DATA_DIR, "THE_MACHINE.yml"))
def test_MachineModel_getter(self):
sample_operands = [
{
'memory': {
'offset': None,
'base': {'name': 'r12'},
'index': {'name': 'rcx'},
'scale': 8,
"memory": {
"offset": None,
"base": {"name": "r12"},
"index": {"name": "rcx"},
"scale": 8,
}
}
]
self.assertIsNone(self.machine_model_csx.get_instruction('GETRESULT', sample_operands))
self.assertIsNone(self.machine_model_tx2.get_instruction('GETRESULT', sample_operands))
self.assertIsNone(self.machine_model_csx.get_instruction("GETRESULT", sample_operands))
self.assertIsNone(self.machine_model_tx2.get_instruction("GETRESULT", sample_operands))
self.assertEqual(self.machine_model_csx.get_arch(), 'csx')
self.assertEqual(self.machine_model_tx2.get_arch(), 'tx2')
self.assertEqual(self.machine_model_csx.get_arch(), "csx")
self.assertEqual(self.machine_model_tx2.get_arch(), "tx2")
self.assertEqual(self.machine_model_csx.get_ISA(), 'x86')
self.assertEqual(self.machine_model_tx2.get_ISA(), 'aarch64')
self.assertEqual(self.machine_model_csx.get_ISA(), "x86")
self.assertEqual(self.machine_model_tx2.get_ISA(), "aarch64")
ports_csx = ['0', '0DV', '1', '2', '2D', '3', '3D', '4', '5', '6', '7']
data_ports_csx = ['2D', '3D']
ports_csx = ["0", "0DV", "1", "2", "2D", "3", "3D", "4", "5", "6", "7"]
data_ports_csx = ["2D", "3D"]
self.assertEqual(self.machine_model_csx.get_ports(), ports_csx)
self.assertEqual(self.machine_model_csx.get_data_ports(), data_ports_csx)
self.assertFalse(self.machine_model_tx2.has_hidden_loads())
self.assertEqual(MachineModel.get_isa_for_arch('CSX'), 'x86')
self.assertEqual(MachineModel.get_isa_for_arch('tX2'), 'aarch64')
self.assertEqual(MachineModel.get_isa_for_arch("CSX"), "x86")
self.assertEqual(MachineModel.get_isa_for_arch("tX2"), "aarch64")
with self.assertRaises(ValueError):
self.assertIsNone(MachineModel.get_isa_for_arch('THE_MACHINE'))
self.assertIsNone(MachineModel.get_isa_for_arch("THE_MACHINE"))
##################
# Helper functions
@@ -492,11 +491,11 @@ class TestSemanticTools(unittest.TestCase):
@staticmethod
def _find_file(name):
testdir = os.path.dirname(__file__)
name = os.path.join(testdir, 'test_files', name)
name = os.path.join(testdir, "test_files", name)
assert os.path.exists(name)
return name
if __name__ == '__main__':
if __name__ == "__main__":
suite = unittest.TestLoader().loadTestsFromTestCase(TestSemanticTools)
unittest.TextTestRunner(verbosity=2).run(suite)