diff --git a/.gitignore b/.gitignore index 2d40989..6ef37fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # OSACA specific files and folders -osaca/taxCalc/ +*.*.pickle # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.travis.yml b/.travis.yml index 3edb959..421377a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ language: python python: - "3.5" - "3.6" -# Python 3.7 not working yet - "3.7" - "3.8" + - "3.9" before_install: # - pip install tox-travis - pip install codecov diff --git a/MANIFEST.in b/MANIFEST.in index f4d516a..1c0ac90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ include README.rst include LICENSE include tox.ini recursive-include osaca/data/ *.yml +recursive-include osaca/data/ *.pickle +include osaca/data/_build_cache.py include examples/* recursive-include tests *.py *.out recursive-include tests/testfiles/ * diff --git a/README.rst b/README.rst index 72a92a5..9d8fde7 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ Open Source Architecture Code Analyzer For an innermost loop kernel in assembly, this tool allows automatic instruction fetching of assembly code and automatic runtime prediction including throughput analysis and detection for critical path and loop-carried dependencies. -.. image:: https://travis-ci.org/RRZE-HPC/OSACA.svg?branch=master - :target: https://travis-ci.org/RRZE-HPC/OSACA +.. image:: https://travis-ci.com/RRZE-HPC/OSACA.svg?branch=master + :target: https://travis-ci.com/github/RRZE-HPC/OSACA :alt: Build Status .. image:: https://codecov.io/github/RRZE-HPC/OSACA/coverage.svg?branch=master diff --git a/osaca/data/_build_cache.py b/osaca/data/_build_cache.py new file mode 100644 index 0000000..fad10ed --- /dev/null +++ b/osaca/data/_build_cache.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from glob import glob +import os.path +import sys +sys.path[0:0] = ['../..'] + +from osaca.semantics.hw_model import MachineModel + +print('Building cache: ', end='') +sys.stdout.flush() + +# Iterating architectures +for f in glob(os.path.join(os.path.dirname(__file__), '*.yml')): + MachineModel(path_to_yaml=f) + print('.', end='') + sys.stdout.flush() + +# Iterating ISAs +for f in glob(os.path.join(os.path.dirname(__file__), 'isa/*.yml')): + MachineModel(path_to_yaml=f) + print('+', end='') + sys.stdout.flush() + +print() \ No newline at end of file diff --git a/osaca/osaca.py b/osaca/osaca.py index 9c82f3c..85c76b0 100755 --- a/osaca/osaca.py +++ b/osaca/osaca.py @@ -12,11 +12,7 @@ from osaca.parser import BaseParser, ParserAArch64, ParserX86ATT from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section) -MODULE_DATA_DIR = os.path.join( - os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' -) -LOCAL_OSACA_DIR = os.path.join(os.path.expanduser('~') + '/.osaca/') -DATA_DIR = os.path.join(LOCAL_OSACA_DIR, 'data/') + SUPPORTED_ARCHS = [ 'SNB', 'IVB', diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index 0bb398c..ca1506c 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import base64 import os import pickle import re import string from copy import deepcopy from itertools import product +import hashlib +from pathlib import Path import ruamel.yaml from ruamel.yaml.compat import StringIO @@ -49,7 +50,7 @@ class MachineModel(object): yaml = self._create_yaml_object() if arch: self._arch = arch.lower() - self._path = utils.find_file(self._arch + '.yml') + self._path = utils.find_datafile(self._arch + '.yml') # check if file is cached cached = self._get_cached(self._path) if not lazy else False if cached: @@ -314,18 +315,22 @@ class MachineModel(object): :type filepath: str :returns: cached DB if existing, `False` otherwise """ - hashname = self._get_hashname(filepath) - cachepath = utils.exists_cached_file(hashname + '.pickle') - if cachepath: - # Check if modification date of DB is older than cached version - if os.path.getmtime(filepath) < os.path.getmtime(cachepath): - # load cached version - with open(cachepath, 'rb') as f: - cached_db = pickle.load(f) - return cached_db - else: - # DB newer than cached version --> delete cached file and return False - os.remove(cachepath) + p = Path(filepath) + # 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix + companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle') + if companion_cachefile.exists(): + if companion_cachefile.stat().st_mtime > p.stat().st_mtime: + # companion file up-to-date + with companion_cachefile.open('rb') as f: + return pickle.load(f) + + # 2. home cachefile: ~/.osaca/cache/.pickle + hexhash = hashlib.sha256(p.read_bytes()).hexdigest() + home_cachefile = (Path(utils.CACHE_DIR) / hexhash).with_suffix('.pickle') + if home_cachefile.exists(): + # home file (must be up-to-date, due to equal hash) + with home_cachefile.open('rb') as f: + return pickle.load(f) return False def _write_in_cache(self, filepath, data): @@ -337,14 +342,25 @@ class MachineModel(object): :param data: :class:`MachineModel` to store :type data: :class:`dict` """ - hashname = self._get_hashname(filepath) - filepath = os.path.join(utils.CACHE_DIR, hashname + '.pickle') - with open(filepath, 'wb') as f: - pickle.dump(data, f) + p = Path(filepath) + # 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix + companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle') + if os.access(str(companion_cachefile.parent), os.W_OK): + with companion_cachefile.open('wb') as f: + pickle.dump(data, f) + return - def _get_hashname(self, name): - """Returns unique hashname for machine model""" - return base64.b64encode(name.encode()).decode() + # 2. home cachefile: ~/.osaca/cache/.pickle + hexhash = hashlib.sha256(p.read_bytes()).hexdigest() + cache_dir = Path(utils.CACHE_DIR) + try: + os.makedirs(cache_dir, exist_ok=True) + except OSError: + return + home_cachefile = (cache_dir / hexhash).with_suffix('.pickle') + if os.access(str(home_cachefile.parent), os.W_OK): + with home_cachefile.open('wb') as f: + pickle.dump(data, f) def _get_key(self, name, operands): """Get unique instruction form key for dict DB.""" diff --git a/osaca/semantics/isa_semantics.py b/osaca/semantics/isa_semantics.py index b624f10..f475009 100755 --- a/osaca/semantics/isa_semantics.py +++ b/osaca/semantics/isa_semantics.py @@ -26,7 +26,7 @@ class ISASemantics(object): def __init__(self, isa, path_to_yaml=None): self._isa = isa.lower() - path = utils.find_file('isa/' + self._isa + '.yml') if not path_to_yaml else path_to_yaml + path = path_to_yaml or utils.find_datafile('isa/' + self._isa + '.yml') self._isa_model = MachineModel(path_to_yaml=path) if self._isa == 'x86': self._parser = ParserX86ATT() diff --git a/osaca/utils.py b/osaca/utils.py index f232167..53ff292 100644 --- a/osaca/utils.py +++ b/osaca/utils.py @@ -1,28 +1,14 @@ #!/usr/bin/env python3 import os.path +DATA_DIRS = [os.path.expanduser('~/.osaca/data'), os.path.join(os.path.dirname(__file__), 'data')] CACHE_DIR = os.path.expanduser('~/.osaca/cache') -def find_file(name): +def find_datafile(name): """Check for existence of name in user or package data folders and return path.""" - search_paths = [os.path.expanduser('~/.osaca/data'), - os.path.join(os.path.dirname(__file__), 'data')] - for dir in search_paths: + for dir in DATA_DIRS: path = os.path.join(dir, name) if os.path.exists(path): return path - raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, search_paths)) - - -def exists_cached_file(name): - """Check for existence of file in cache dir. Returns path if it exists and False otherwise.""" - if not os.path.exists(CACHE_DIR): - os.makedirs(CACHE_DIR) - return False - search_paths = [CACHE_DIR] - for dir in search_paths: - path = os.path.join(dir, name) - if os.path.exists(path): - return path - return False + raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, DATA_DIRS)) diff --git a/setup.py b/setup.py index 2e380b8..38785b0 100755 --- a/setup.py +++ b/setup.py @@ -2,11 +2,14 @@ # Always prefer setuptools over distutils from setuptools import setup, find_packages +from setuptools.command.install import install as _install +from setuptools.command.sdist import sdist as _sdist # To use a consistent encoding from codecs import open import os import io import re +import sys here = os.path.abspath(os.path.dirname(__file__)) @@ -27,6 +30,27 @@ def find_version(*file_paths): raise RuntimeError("Unable to find version string.") +def _run_build_cache(dir): + from subprocess import check_call + # This is run inside the install staging directory (that had no .pyc files) + # We don't want to generate any. + # https://github.com/eliben/pycparser/pull/135 + check_call([sys.executable, '-B', '_build_cache.py'], + cwd=os.path.join(dir, 'osaca', 'data')) + + +class install(_install): + def run(self): + _install.run(self) + self.execute(_run_build_cache, (self.install_lib,), msg="Build ISA and architecture cache") + + +class sdist(_sdist): + def make_release_tree(self, basedir, files): + _sdist.make_release_tree(self, basedir, files) + self.execute(_run_build_cache, (basedir,), msg="Build ISA and architecture cache") + + # Get the long description from the README file with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() @@ -59,7 +83,7 @@ setup( # 3 - Alpha # 4 - Beta # 5 - Production/Stable - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', # Indicate who your project is intended for 'Intended Audience :: Developers', @@ -76,6 +100,9 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], # What doesd your project relate to? @@ -124,4 +151,7 @@ setup( 'osaca=osaca.osaca:main', ], }, + + # Overwriting install and sdist to enforce cache distribution with package + cmdclass={'install': install, 'sdist': sdist}, ) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index eb1ca86..9c2ef46 100755 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -33,7 +33,7 @@ class TestFrontend(unittest.TestCase): path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') ) self.machine_model_tx2 = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') + arch='tx2' ) self.semantics_csx = ArchSemantics( self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') diff --git a/tests/test_semantics.py b/tests/test_semantics.py index d8ffa26..72f1678 100755 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -20,48 +20,43 @@ class TestSemanticTools(unittest.TestCase): MODULE_DATA_DIR = os.path.join( os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' ) - USER_DATA_DIR = os.path.join(os.path.expanduser('~'), '.osaca/') @classmethod - def setUpClass(self): - # copy db files in user directory - if not os.path.isdir(os.path.join(self.USER_DATA_DIR, 'data')): - os.makedirs(os.path.join(self.USER_DATA_DIR, 'data')) - call(['cp', '-r', self.MODULE_DATA_DIR, self.USER_DATA_DIR]) + def setUpClass(cls): # set up parser and kernels - self.parser_x86 = ParserX86ATT() - self.parser_AArch64 = ParserAArch64() - with open(self._find_file('kernel_x86.s')) as f: - self.code_x86 = f.read() - with open(self._find_file('kernel_aarch64.s')) as f: - self.code_AArch64 = f.read() - self.kernel_x86 = reduce_to_section(self.parser_x86.parse_file(self.code_x86), 'x86') - self.kernel_AArch64 = reduce_to_section( - self.parser_AArch64.parse_file(self.code_AArch64), 'aarch64' + cls.parser_x86 = ParserX86ATT() + cls.parser_AArch64 = ParserAArch64() + with open(cls._find_file('kernel_x86.s')) as f: + cls.code_x86 = f.read() + with open(cls._find_file('kernel_aarch64.s')) as f: + cls.code_AArch64 = f.read() + cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), 'x86') + cls.kernel_AArch64 = reduce_to_section( + cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64' ) # set up machine models - self.machine_model_csx = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') + cls.machine_model_csx = MachineModel( + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml') ) - self.machine_model_tx2 = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') + cls.machine_model_tx2 = MachineModel( + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml') ) - self.semantics_csx = ArchSemantics( - self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') + cls.semantics_csx = ArchSemantics( + cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/x86.yml') ) - self.semantics_tx2 = ArchSemantics( - self.machine_model_tx2, - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'), + cls.semantics_tx2 = ArchSemantics( + cls.machine_model_tx2, + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'), ) - self.machine_model_zen = MachineModel(arch='zen1') + cls.machine_model_zen = MachineModel(arch='zen1') - for i in range(len(self.kernel_x86)): - self.semantics_csx.assign_src_dst(self.kernel_x86[i]) - self.semantics_csx.assign_tp_lt(self.kernel_x86[i]) - for i in range(len(self.kernel_AArch64)): - self.semantics_tx2.assign_src_dst(self.kernel_AArch64[i]) - self.semantics_tx2.assign_tp_lt(self.kernel_AArch64[i]) + for i in range(len(cls.kernel_x86)): + cls.semantics_csx.assign_src_dst(cls.kernel_x86[i]) + cls.semantics_csx.assign_tp_lt(cls.kernel_x86[i]) + for i in range(len(cls.kernel_AArch64)): + cls.semantics_tx2.assign_src_dst(cls.kernel_AArch64[i]) + cls.semantics_tx2.assign_tp_lt(cls.kernel_AArch64[i]) ########### # Tests diff --git a/tox.ini b/tox.ini index 37ea5e3..6af586d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35,py36 +envlist = py35,py36,py37,py38,py39 [testenv] commands= python tests/all_tests.py