Lerking/OSACA
1
0
Fork 0
mirror of https://github.com/RRZE-HPC/OSACA.git synced 2026-01-08 12:10:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/RRZE-HPC/OSACA

Browse Source
This commit is contained in:
JanLJL
2020-10-29 13:00:09 +01:00
parent 022598d94f 1f5c9d1c61
commit 1c889fa785
13 changed files with 132 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,5 +1,5 @@
# OSACA specific files and folders
osaca/taxCalc/
*.*.pickle
# Byte-compiled / optimized / DLL files
__pycache__/

2
.travis.yml
View File

@@ -3,9 +3,9 @@ language: python
python:
- "3.5"
- "3.6"
# Python 3.7 not working yet
- "3.7"
- "3.8"
- "3.9"
before_install:
# - pip install tox-travis
- pip install codecov

2
MANIFEST.in
View File

@@ -2,6 +2,8 @@ include README.rst
include LICENSE
include tox.ini
recursive-include osaca/data/ *.yml
recursive-include osaca/data/ *.pickle
include osaca/data/_build_cache.py
include examples/*
recursive-include tests *.py *.out
recursive-include tests/testfiles/ *

4
README.rst
View File

@@ -10,8 +10,8 @@ Open Source Architecture Code Analyzer
For an innermost loop kernel in assembly, this tool allows automatic instruction fetching of assembly code and automatic runtime prediction including throughput analysis and detection for critical path and loop-carried dependencies.
.. image:: https://travis-ci.org/RRZE-HPC/OSACA.svg?branch=master
:target: https://travis-ci.org/RRZE-HPC/OSACA
.. image:: https://travis-ci.com/RRZE-HPC/OSACA.svg?branch=master
:target: https://travis-ci.com/github/RRZE-HPC/OSACA
:alt: Build Status
.. image:: https://codecov.io/github/RRZE-HPC/OSACA/coverage.svg?branch=master

24
osaca/data/_build_cache.py Normal file
View File

@@ -0,0 +1,24 @@
#!/usr/bin/env python3
from glob import glob
import os.path
import sys
sys.path[0:0] = ['../..']
from osaca.semantics.hw_model import MachineModel
print('Building cache: ', end='')
sys.stdout.flush()
# Iterating architectures
for f in glob(os.path.join(os.path.dirname(__file__), '*.yml')):
MachineModel(path_to_yaml=f)
print('.', end='')
sys.stdout.flush()
# Iterating ISAs
for f in glob(os.path.join(os.path.dirname(__file__), 'isa/*.yml')):
MachineModel(path_to_yaml=f)
print('+', end='')
sys.stdout.flush()
print()

6
osaca/osaca.py
View File

@@ -12,11 +12,7 @@ from osaca.parser import BaseParser, ParserAArch64, ParserX86ATT
from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG,
MachineModel, reduce_to_section)
MODULE_DATA_DIR = os.path.join(
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/'
)
LOCAL_OSACA_DIR = os.path.join(os.path.expanduser('~') + '/.osaca/')
DATA_DIR = os.path.join(LOCAL_OSACA_DIR, 'data/')
SUPPORTED_ARCHS = [
'SNB',
'IVB',

58
osaca/semantics/hw_model.py
View File

@@ -1,12 +1,13 @@
#!/usr/bin/env python3
import base64
import os
import pickle
import re
import string
from copy import deepcopy
from itertools import product
import hashlib
from pathlib import Path
import ruamel.yaml
from ruamel.yaml.compat import StringIO
@@ -49,7 +50,7 @@ class MachineModel(object):
yaml = self._create_yaml_object()
if arch:
self._arch = arch.lower()
self._path = utils.find_file(self._arch + '.yml')
self._path = utils.find_datafile(self._arch + '.yml')
# check if file is cached
cached = self._get_cached(self._path) if not lazy else False
if cached:
@@ -314,18 +315,22 @@ class MachineModel(object):
:type filepath: str
:returns: cached DB if existing, `False` otherwise
"""
hashname = self._get_hashname(filepath)
cachepath = utils.exists_cached_file(hashname + '.pickle')
if cachepath:
# Check if modification date of DB is older than cached version
if os.path.getmtime(filepath) < os.path.getmtime(cachepath):
# load cached version
with open(cachepath, 'rb') as f:
cached_db = pickle.load(f)
return cached_db
else:
# DB newer than cached version --> delete cached file and return False
os.remove(cachepath)
p = Path(filepath)
# 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix
companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle')
if companion_cachefile.exists():
if companion_cachefile.stat().st_mtime > p.stat().st_mtime:
# companion file up-to-date
with companion_cachefile.open('rb') as f:
return pickle.load(f)
# 2. home cachefile: ~/.osaca/cache/<sha512hash>.pickle
hexhash = hashlib.sha256(p.read_bytes()).hexdigest()
home_cachefile = (Path(utils.CACHE_DIR) / hexhash).with_suffix('.pickle')
if home_cachefile.exists():
# home file (must be up-to-date, due to equal hash)
with home_cachefile.open('rb') as f:
return pickle.load(f)
return False
def _write_in_cache(self, filepath, data):
@@ -337,14 +342,25 @@ class MachineModel(object):
:param data: :class:`MachineModel` to store
:type data: :class:`dict`
"""
hashname = self._get_hashname(filepath)
filepath = os.path.join(utils.CACHE_DIR, hashname + '.pickle')
with open(filepath, 'wb') as f:
pickle.dump(data, f)
p = Path(filepath)
# 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix
companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle')
if os.access(str(companion_cachefile.parent), os.W_OK):
with companion_cachefile.open('wb') as f:
pickle.dump(data, f)
return
def _get_hashname(self, name):
"""Returns unique hashname for machine model"""
return base64.b64encode(name.encode()).decode()
# 2. home cachefile: ~/.osaca/cache/<sha512hash>.pickle
hexhash = hashlib.sha256(p.read_bytes()).hexdigest()
cache_dir = Path(utils.CACHE_DIR)
try:
os.makedirs(cache_dir, exist_ok=True)
except OSError:
return
home_cachefile = (cache_dir / hexhash).with_suffix('.pickle')
if os.access(str(home_cachefile.parent), os.W_OK):
with home_cachefile.open('wb') as f:
pickle.dump(data, f)
def _get_key(self, name, operands):
"""Get unique instruction form key for dict DB."""

2
osaca/semantics/isa_semantics.py
View File

@@ -26,7 +26,7 @@ class ISASemantics(object):
def __init__(self, isa, path_to_yaml=None):
self._isa = isa.lower()
path = utils.find_file('isa/' + self._isa + '.yml') if not path_to_yaml else path_to_yaml
path = path_to_yaml or utils.find_datafile('isa/' + self._isa + '.yml')
self._isa_model = MachineModel(path_to_yaml=path)
if self._isa == 'x86':
self._parser = ParserX86ATT()

22
osaca/utils.py
View File

@@ -1,28 +1,14 @@
#!/usr/bin/env python3
import os.path
DATA_DIRS = [os.path.expanduser('~/.osaca/data'), os.path.join(os.path.dirname(__file__), 'data')]
CACHE_DIR = os.path.expanduser('~/.osaca/cache')
def find_file(name):
def find_datafile(name):
"""Check for existence of name in user or package data folders and return path."""
search_paths = [os.path.expanduser('~/.osaca/data'),
os.path.join(os.path.dirname(__file__), 'data')]
for dir in search_paths:
for dir in DATA_DIRS:
path = os.path.join(dir, name)
if os.path.exists(path):
return path
raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, search_paths))
def exists_cached_file(name):
"""Check for existence of file in cache dir. Returns path if it exists and False otherwise."""
if not os.path.exists(CACHE_DIR):
os.makedirs(CACHE_DIR)
return False
search_paths = [CACHE_DIR]
for dir in search_paths:
path = os.path.join(dir, name)
if os.path.exists(path):
return path
return False
raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, DATA_DIRS))

32
setup.py
View File

@@ -2,11 +2,14 @@
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
from setuptools.command.install import install as _install
from setuptools.command.sdist import sdist as _sdist
# To use a consistent encoding
from codecs import open
import os
import io
import re
import sys
here = os.path.abspath(os.path.dirname(__file__))
@@ -27,6 +30,27 @@ def find_version(*file_paths):
raise RuntimeError("Unable to find version string.")
def _run_build_cache(dir):
from subprocess import check_call
# This is run inside the install staging directory (that had no .pyc files)
# We don't want to generate any.
# https://github.com/eliben/pycparser/pull/135
check_call([sys.executable, '-B', '_build_cache.py'],
cwd=os.path.join(dir, 'osaca', 'data'))
class install(_install):
def run(self):
_install.run(self)
self.execute(_run_build_cache, (self.install_lib,), msg="Build ISA and architecture cache")
class sdist(_sdist):
def make_release_tree(self, basedir, files):
_sdist.make_release_tree(self, basedir, files)
self.execute(_run_build_cache, (basedir,), msg="Build ISA and architecture cache")
# Get the long description from the README file
with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
@@ -59,7 +83,7 @@ setup(
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 3 - Alpha',
'Development Status :: 4 - Beta',
# Indicate who your project is intended for
'Intended Audience :: Developers',
@@ -76,6 +100,9 @@ setup(
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
],
# What doesd your project relate to?
@@ -124,4 +151,7 @@ setup(
'osaca=osaca.osaca:main',
],
},
# Overwriting install and sdist to enforce cache distribution with package
cmdclass={'install': install, 'sdist': sdist},
)

2
tests/test_frontend.py
View File

@@ -33,7 +33,7 @@ class TestFrontend(unittest.TestCase):
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')
)
self.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml')
arch='tx2'
)
self.semantics_csx = ArchSemantics(
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml')

57
tests/test_semantics.py
View File

@@ -20,48 +20,43 @@ class TestSemanticTools(unittest.TestCase):
MODULE_DATA_DIR = os.path.join(
os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/'
)
USER_DATA_DIR = os.path.join(os.path.expanduser('~'), '.osaca/')
@classmethod
def setUpClass(self):
# copy db files in user directory
if not os.path.isdir(os.path.join(self.USER_DATA_DIR, 'data')):
os.makedirs(os.path.join(self.USER_DATA_DIR, 'data'))
call(['cp', '-r', self.MODULE_DATA_DIR, self.USER_DATA_DIR])
def setUpClass(cls):
# set up parser and kernels
self.parser_x86 = ParserX86ATT()
self.parser_AArch64 = ParserAArch64()
with open(self._find_file('kernel_x86.s')) as f:
self.code_x86 = f.read()
with open(self._find_file('kernel_aarch64.s')) as f:
self.code_AArch64 = f.read()
self.kernel_x86 = reduce_to_section(self.parser_x86.parse_file(self.code_x86), 'x86')
self.kernel_AArch64 = reduce_to_section(
self.parser_AArch64.parse_file(self.code_AArch64), 'aarch64'
cls.parser_x86 = ParserX86ATT()
cls.parser_AArch64 = ParserAArch64()
with open(cls._find_file('kernel_x86.s')) as f:
cls.code_x86 = f.read()
with open(cls._find_file('kernel_aarch64.s')) as f:
cls.code_AArch64 = f.read()
cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), 'x86')
cls.kernel_AArch64 = reduce_to_section(
cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64'
)
# set up machine models
self.machine_model_csx = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml')
cls.machine_model_csx = MachineModel(
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml')
)
self.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml')
cls.machine_model_tx2 = MachineModel(
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml')
)
self.semantics_csx = ArchSemantics(
self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml')
cls.semantics_csx = ArchSemantics(
cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/x86.yml')
)
self.semantics_tx2 = ArchSemantics(
self.machine_model_tx2,
path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'),
cls.semantics_tx2 = ArchSemantics(
cls.machine_model_tx2,
path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'),
)
self.machine_model_zen = MachineModel(arch='zen1')
cls.machine_model_zen = MachineModel(arch='zen1')
for i in range(len(self.kernel_x86)):
self.semantics_csx.assign_src_dst(self.kernel_x86[i])
self.semantics_csx.assign_tp_lt(self.kernel_x86[i])
for i in range(len(self.kernel_AArch64)):
self.semantics_tx2.assign_src_dst(self.kernel_AArch64[i])
self.semantics_tx2.assign_tp_lt(self.kernel_AArch64[i])
for i in range(len(cls.kernel_x86)):
cls.semantics_csx.assign_src_dst(cls.kernel_x86[i])
cls.semantics_csx.assign_tp_lt(cls.kernel_x86[i])
for i in range(len(cls.kernel_AArch64)):
cls.semantics_tx2.assign_src_dst(cls.kernel_AArch64[i])
cls.semantics_tx2.assign_tp_lt(cls.kernel_AArch64[i])
###########
# Tests

2
tox.ini
View File

@@ -1,5 +1,5 @@
[tox]
envlist = py35,py36
envlist = py35,py36,py37,py38,py39
[testenv]
commands=
python tests/all_tests.py