From 9d2ea8603f706cdddf240ce64b9286db488d5977 Mon Sep 17 00:00:00 2001 From: Julian Hammer Date: Wed, 28 Oct 2020 16:29:23 +0100 Subject: [PATCH 1/6] new caching structure with support for distribution --- osaca/osaca.py | 6 +--- osaca/semantics/hw_model.py | 57 +++++++++++++++++++++----------- osaca/semantics/isa_semantics.py | 2 +- osaca/utils.py | 25 ++++---------- tests/test_frontend.py | 2 +- tests/test_semantics.py | 57 +++++++++++++++----------------- 6 files changed, 73 insertions(+), 76 deletions(-) diff --git a/osaca/osaca.py b/osaca/osaca.py index 656ad88..7ed234c 100755 --- a/osaca/osaca.py +++ b/osaca/osaca.py @@ -12,11 +12,7 @@ from osaca.parser import BaseParser, ParserAArch64, ParserX86ATT from osaca.semantics import (INSTR_FLAGS, ArchSemantics, KernelDG, MachineModel, reduce_to_section) -MODULE_DATA_DIR = os.path.join( - os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' -) -LOCAL_OSACA_DIR = os.path.join(os.path.expanduser('~') + '/.osaca/') -DATA_DIR = os.path.join(LOCAL_OSACA_DIR, 'data/') + SUPPORTED_ARCHS = [ 'SNB', 'IVB', diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index 0bb398c..6e55cc1 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -7,6 +7,8 @@ import re import string from copy import deepcopy from itertools import product +import hashlib +from pathlib import Path import ruamel.yaml from ruamel.yaml.compat import StringIO @@ -49,7 +51,7 @@ class MachineModel(object): yaml = self._create_yaml_object() if arch: self._arch = arch.lower() - self._path = utils.find_file(self._arch + '.yml') + self._path = utils.find_datafile(self._arch + '.yml') # check if file is cached cached = self._get_cached(self._path) if not lazy else False if cached: @@ -314,18 +316,22 @@ class MachineModel(object): :type filepath: str :returns: cached DB if existing, `False` otherwise """ - hashname = self._get_hashname(filepath) - cachepath = utils.exists_cached_file(hashname + '.pickle') - if cachepath: - # Check if modification date of DB is older than cached version - if os.path.getmtime(filepath) < os.path.getmtime(cachepath): - # load cached version - with open(cachepath, 'rb') as f: - cached_db = pickle.load(f) - return cached_db - else: - # DB newer than cached version --> delete cached file and return False - os.remove(cachepath) + p = Path(filepath) + # 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix + companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle') + if companion_cachefile.exists(): + if companion_cachefile.stat().st_mtime > p.stat().st_mtime: + # companion file up-to-date + with companion_cachefile.open('rb') as f: + return pickle.load(f) + + # 2. home cachefile: ~/.osaca/cache/.pickle + hexhash = hashlib.sha256(p.read_bytes()).hexdigest() + home_cachefile = (Path(utils.CACHE_DIR) / hexhash).with_suffix('.pickle') + if home_cachefile.exists(): + # home file (must be up-to-date, due to equal hash) + with home_cachefile.open('rb') as f: + return pickle.load(f) return False def _write_in_cache(self, filepath, data): @@ -337,14 +343,25 @@ class MachineModel(object): :param data: :class:`MachineModel` to store :type data: :class:`dict` """ - hashname = self._get_hashname(filepath) - filepath = os.path.join(utils.CACHE_DIR, hashname + '.pickle') - with open(filepath, 'wb') as f: - pickle.dump(data, f) + p = Path(filepath) + # 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix + companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle') + if os.access(companion_cachefile.parent, os.W_OK): + with companion_cachefile.open('wb') as f: + pickle.dump(data, f) + return - def _get_hashname(self, name): - """Returns unique hashname for machine model""" - return base64.b64encode(name.encode()).decode() + # 2. home cachefile: ~/.osaca/cache/.pickle + hexhash = hashlib.sha256(p.read_bytes()).hexdigest() + cache_dir = Path(utils.CACHE_DIR) + try: + os.makedirs(cache_dir, exist_ok=True) + except OSError: + return + home_cachefile = (cache_dir / hexhash).with_suffix('.pickle') + if os.access(home_cachefile.parent, os.W_OK): + with home_cachefile.open('wb') as f: + pickle.dump(data, f) def _get_key(self, name, operands): """Get unique instruction form key for dict DB.""" diff --git a/osaca/semantics/isa_semantics.py b/osaca/semantics/isa_semantics.py index b624f10..f475009 100755 --- a/osaca/semantics/isa_semantics.py +++ b/osaca/semantics/isa_semantics.py @@ -26,7 +26,7 @@ class ISASemantics(object): def __init__(self, isa, path_to_yaml=None): self._isa = isa.lower() - path = utils.find_file('isa/' + self._isa + '.yml') if not path_to_yaml else path_to_yaml + path = path_to_yaml or utils.find_datafile('isa/' + self._isa + '.yml') self._isa_model = MachineModel(path_to_yaml=path) if self._isa == 'x86': self._parser = ParserX86ATT() diff --git a/osaca/utils.py b/osaca/utils.py index f232167..36529e3 100644 --- a/osaca/utils.py +++ b/osaca/utils.py @@ -1,28 +1,17 @@ #!/usr/bin/env python3 import os.path +from pathlib import Path +import hashlib +DATA_DIRS = [os.path.expanduser('~/.osaca/data'), + os.path.join(os.path.dirname(__file__), 'data')] CACHE_DIR = os.path.expanduser('~/.osaca/cache') -def find_file(name): +def find_datafile(name): """Check for existence of name in user or package data folders and return path.""" - search_paths = [os.path.expanduser('~/.osaca/data'), - os.path.join(os.path.dirname(__file__), 'data')] - for dir in search_paths: + for dir in DATA_DIRS: path = os.path.join(dir, name) if os.path.exists(path): return path - raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, search_paths)) - - -def exists_cached_file(name): - """Check for existence of file in cache dir. Returns path if it exists and False otherwise.""" - if not os.path.exists(CACHE_DIR): - os.makedirs(CACHE_DIR) - return False - search_paths = [CACHE_DIR] - for dir in search_paths: - path = os.path.join(dir, name) - if os.path.exists(path): - return path - return False + raise FileNotFoundError("Could not find {!r} in {!r}.".format(name, DATA_DIRS)) diff --git a/tests/test_frontend.py b/tests/test_frontend.py index eb1ca86..f185a03 100755 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -33,7 +33,7 @@ class TestFrontend(unittest.TestCase): path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') ) self.machine_model_tx2 = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') + arch='tx2.yml' ) self.semantics_csx = ArchSemantics( self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') diff --git a/tests/test_semantics.py b/tests/test_semantics.py index d8ffa26..72f1678 100755 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -20,48 +20,43 @@ class TestSemanticTools(unittest.TestCase): MODULE_DATA_DIR = os.path.join( os.path.dirname(os.path.split(os.path.abspath(__file__))[0]), 'osaca/data/' ) - USER_DATA_DIR = os.path.join(os.path.expanduser('~'), '.osaca/') @classmethod - def setUpClass(self): - # copy db files in user directory - if not os.path.isdir(os.path.join(self.USER_DATA_DIR, 'data')): - os.makedirs(os.path.join(self.USER_DATA_DIR, 'data')) - call(['cp', '-r', self.MODULE_DATA_DIR, self.USER_DATA_DIR]) + def setUpClass(cls): # set up parser and kernels - self.parser_x86 = ParserX86ATT() - self.parser_AArch64 = ParserAArch64() - with open(self._find_file('kernel_x86.s')) as f: - self.code_x86 = f.read() - with open(self._find_file('kernel_aarch64.s')) as f: - self.code_AArch64 = f.read() - self.kernel_x86 = reduce_to_section(self.parser_x86.parse_file(self.code_x86), 'x86') - self.kernel_AArch64 = reduce_to_section( - self.parser_AArch64.parse_file(self.code_AArch64), 'aarch64' + cls.parser_x86 = ParserX86ATT() + cls.parser_AArch64 = ParserAArch64() + with open(cls._find_file('kernel_x86.s')) as f: + cls.code_x86 = f.read() + with open(cls._find_file('kernel_aarch64.s')) as f: + cls.code_AArch64 = f.read() + cls.kernel_x86 = reduce_to_section(cls.parser_x86.parse_file(cls.code_x86), 'x86') + cls.kernel_AArch64 = reduce_to_section( + cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64' ) # set up machine models - self.machine_model_csx = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') + cls.machine_model_csx = MachineModel( + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml') ) - self.machine_model_tx2 = MachineModel( - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'tx2.yml') + cls.machine_model_tx2 = MachineModel( + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml') ) - self.semantics_csx = ArchSemantics( - self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') + cls.semantics_csx = ArchSemantics( + cls.machine_model_csx, path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/x86.yml') ) - self.semantics_tx2 = ArchSemantics( - self.machine_model_tx2, - path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'), + cls.semantics_tx2 = ArchSemantics( + cls.machine_model_tx2, + path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'), ) - self.machine_model_zen = MachineModel(arch='zen1') + cls.machine_model_zen = MachineModel(arch='zen1') - for i in range(len(self.kernel_x86)): - self.semantics_csx.assign_src_dst(self.kernel_x86[i]) - self.semantics_csx.assign_tp_lt(self.kernel_x86[i]) - for i in range(len(self.kernel_AArch64)): - self.semantics_tx2.assign_src_dst(self.kernel_AArch64[i]) - self.semantics_tx2.assign_tp_lt(self.kernel_AArch64[i]) + for i in range(len(cls.kernel_x86)): + cls.semantics_csx.assign_src_dst(cls.kernel_x86[i]) + cls.semantics_csx.assign_tp_lt(cls.kernel_x86[i]) + for i in range(len(cls.kernel_AArch64)): + cls.semantics_tx2.assign_src_dst(cls.kernel_AArch64[i]) + cls.semantics_tx2.assign_tp_lt(cls.kernel_AArch64[i]) ########### # Tests From a6cb09cf1fc1e474593fa112574f1eae0827f58a Mon Sep 17 00:00:00 2001 From: Julian Hammer Date: Wed, 28 Oct 2020 17:16:03 +0100 Subject: [PATCH 2/6] added cache files to package and building during setup --- MANIFEST.in | 2 ++ osaca/data/_build_cache.py | 24 ++++++++++++++++++++++++ setup.py | 27 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 osaca/data/_build_cache.py diff --git a/MANIFEST.in b/MANIFEST.in index f4d516a..1c0ac90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ include README.rst include LICENSE include tox.ini recursive-include osaca/data/ *.yml +recursive-include osaca/data/ *.pickle +include osaca/data/_build_cache.py include examples/* recursive-include tests *.py *.out recursive-include tests/testfiles/ * diff --git a/osaca/data/_build_cache.py b/osaca/data/_build_cache.py new file mode 100644 index 0000000..fad10ed --- /dev/null +++ b/osaca/data/_build_cache.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from glob import glob +import os.path +import sys +sys.path[0:0] = ['../..'] + +from osaca.semantics.hw_model import MachineModel + +print('Building cache: ', end='') +sys.stdout.flush() + +# Iterating architectures +for f in glob(os.path.join(os.path.dirname(__file__), '*.yml')): + MachineModel(path_to_yaml=f) + print('.', end='') + sys.stdout.flush() + +# Iterating ISAs +for f in glob(os.path.join(os.path.dirname(__file__), 'isa/*.yml')): + MachineModel(path_to_yaml=f) + print('+', end='') + sys.stdout.flush() + +print() \ No newline at end of file diff --git a/setup.py b/setup.py index 2e380b8..ead2243 100755 --- a/setup.py +++ b/setup.py @@ -2,11 +2,14 @@ # Always prefer setuptools over distutils from setuptools import setup, find_packages +from setuptools.command.install import install as _install +from setuptools.command.sdist import sdist as _sdist # To use a consistent encoding from codecs import open import os import io import re +import sys here = os.path.abspath(os.path.dirname(__file__)) @@ -27,6 +30,27 @@ def find_version(*file_paths): raise RuntimeError("Unable to find version string.") +def _run_build_cache(dir): + from subprocess import check_call + # This is run inside the install staging directory (that had no .pyc files) + # We don't want to generate any. + # https://github.com/eliben/pycparser/pull/135 + check_call([sys.executable, '-B', '_build_cache.py'], + cwd=os.path.join(dir, 'osaca', 'data')) + + +class install(_install): + def run(self): + _install.run(self) + self.execute(_run_build_cache, (self.install_lib,), msg="Build ISA and architecture cache") + + +class sdist(_sdist): + def make_release_tree(self, basedir, files): + _sdist.make_release_tree(self, basedir, files) + self.execute(_run_build_cache, (basedir,), msg="Build ISA and architecture cache") + + # Get the long description from the README file with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() @@ -124,4 +148,7 @@ setup( 'osaca=osaca.osaca:main', ], }, + + # Overwriting install and sdist to enforce cache distribution with package + cmdclass={'install': install, 'sdist': sdist}, ) From 9af689b28caf7331bef57b547c9bf70a8eda2309 Mon Sep 17 00:00:00 2001 From: JanLJL Date: Wed, 28 Oct 2020 19:29:48 +0100 Subject: [PATCH 3/6] fixed bug in tests and removed unused imports --- osaca/semantics/hw_model.py | 1 - osaca/utils.py | 5 +---- tests/test_frontend.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index 6e55cc1..d1863c2 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import base64 import os import pickle import re diff --git a/osaca/utils.py b/osaca/utils.py index 36529e3..53ff292 100644 --- a/osaca/utils.py +++ b/osaca/utils.py @@ -1,10 +1,7 @@ #!/usr/bin/env python3 import os.path -from pathlib import Path -import hashlib -DATA_DIRS = [os.path.expanduser('~/.osaca/data'), - os.path.join(os.path.dirname(__file__), 'data')] +DATA_DIRS = [os.path.expanduser('~/.osaca/data'), os.path.join(os.path.dirname(__file__), 'data')] CACHE_DIR = os.path.expanduser('~/.osaca/cache') diff --git a/tests/test_frontend.py b/tests/test_frontend.py index f185a03..9c2ef46 100755 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -33,7 +33,7 @@ class TestFrontend(unittest.TestCase): path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml') ) self.machine_model_tx2 = MachineModel( - arch='tx2.yml' + arch='tx2' ) self.semantics_csx = ArchSemantics( self.machine_model_csx, path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/x86.yml') From decec86e5633829162536c08c479d92c9478aa6c Mon Sep 17 00:00:00 2001 From: Julian Hammer Date: Thu, 29 Oct 2020 10:59:00 +0100 Subject: [PATCH 4/6] fixed py3.5 compatability --- osaca/semantics/hw_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osaca/semantics/hw_model.py b/osaca/semantics/hw_model.py index d1863c2..ca1506c 100755 --- a/osaca/semantics/hw_model.py +++ b/osaca/semantics/hw_model.py @@ -345,7 +345,7 @@ class MachineModel(object): p = Path(filepath) # 1. companion cachefile: same location, with '.' prefix and '.pickle' suffix companion_cachefile = p.with_name('.' + p.name).with_suffix('.pickle') - if os.access(companion_cachefile.parent, os.W_OK): + if os.access(str(companion_cachefile.parent), os.W_OK): with companion_cachefile.open('wb') as f: pickle.dump(data, f) return @@ -358,7 +358,7 @@ class MachineModel(object): except OSError: return home_cachefile = (cache_dir / hexhash).with_suffix('.pickle') - if os.access(home_cachefile.parent, os.W_OK): + if os.access(str(home_cachefile.parent), os.W_OK): with home_cachefile.open('wb') as f: pickle.dump(data, f) From 30e0ad038d7a158a9656c22c79085b614535cfb5 Mon Sep 17 00:00:00 2001 From: JanLJL Date: Thu, 29 Oct 2020 11:06:20 +0100 Subject: [PATCH 5/6] ignore pickles in data/ and support py3.9 --- .gitignore | 2 +- .travis.yml | 2 +- setup.py | 5 ++++- tox.ini | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 2d40989..6ef37fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # OSACA specific files and folders -osaca/taxCalc/ +*.*.pickle # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.travis.yml b/.travis.yml index 3edb959..421377a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,9 @@ language: python python: - "3.5" - "3.6" -# Python 3.7 not working yet - "3.7" - "3.8" + - "3.9" before_install: # - pip install tox-travis - pip install codecov diff --git a/setup.py b/setup.py index ead2243..38785b0 100755 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setup( # 3 - Alpha # 4 - Beta # 5 - Production/Stable - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', # Indicate who your project is intended for 'Intended Audience :: Developers', @@ -100,6 +100,9 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], # What doesd your project relate to? diff --git a/tox.ini b/tox.ini index 37ea5e3..6af586d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35,py36 +envlist = py35,py36,py37,py38,py39 [testenv] commands= python tests/all_tests.py From 1f5c9d1c6149a36736c48bc0876079b1cd930e10 Mon Sep 17 00:00:00 2001 From: Julian Date: Thu, 29 Oct 2020 12:45:39 +0100 Subject: [PATCH 6/6] using travis-ci.com badge --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 72a92a5..9d8fde7 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ Open Source Architecture Code Analyzer For an innermost loop kernel in assembly, this tool allows automatic instruction fetching of assembly code and automatic runtime prediction including throughput analysis and detection for critical path and loop-carried dependencies. -.. image:: https://travis-ci.org/RRZE-HPC/OSACA.svg?branch=master - :target: https://travis-ci.org/RRZE-HPC/OSACA +.. image:: https://travis-ci.com/RRZE-HPC/OSACA.svg?branch=master + :target: https://travis-ci.com/github/RRZE-HPC/OSACA :alt: Build Status .. image:: https://codecov.io/github/RRZE-HPC/OSACA/coverage.svg?branch=master