diff --git a/python_appimage/manylinux/download.py b/python_appimage/manylinux/download.py index 6373931..8f439dd 100644 --- a/python_appimage/manylinux/download.py +++ b/python_appimage/manylinux/download.py @@ -10,6 +10,7 @@ import tempfile from typing import List, Optional from .config import Arch, LinuxTag +from ..utils.deps import CACHE_DIR from ..utils.log import debug, log @@ -52,12 +53,16 @@ class Downloader: object.__setattr__(self, 'image', image) + def default_destination(self): + return Path(CACHE_DIR) / f'share/images/{self.image}' + + def download( self, destination: Optional[Path]=None, tag: Optional[str] = 'latest'): - destination = destination or Path(self.image) + destination = destination or self.default_destination() # Authenticate to quay.io. repository = f'pypa/{self.image}' @@ -88,9 +93,26 @@ class Downloader: # Check missing layers to download. required = [layer['digest'].split(':', 1)[-1] for layer in manifest['layers']] - is_missing = lambda hash_: \ - not (destination / f'layers/{hash_}.tar.gz').exists() - missing = tuple(filter(is_missing, required)) + + missing = [] + for hash_ in required: + path = destination / f'layers/{hash_}.tar.gz' + if path.exists(): + hasher = hashlib.sha256() + with path.open('rb') as f: + while True: + chunk = f.read(CHUNK_SIZE) + if not chunk: + break + else: + hasher.update(chunk) + h = hasher.hexdigest() + if h != hash_: + missing.append(hash_) + else: + debug('FOUND', f'{hash_}.tar.gz') + else: + missing.append(hash_) # Fetch missing layers. with tempfile.TemporaryDirectory() as tmpdir: diff --git a/python_appimage/manylinux/extract.py b/python_appimage/manylinux/extract.py index 6202aa7..a157867 100644 --- a/python_appimage/manylinux/extract.py +++ b/python_appimage/manylinux/extract.py @@ -1,3 +1,4 @@ +import atexit from dataclasses import dataclass, field from distutils.version import LooseVersion import glob @@ -83,6 +84,10 @@ class PythonExtractor: if ssl: paths.append(Path(ssl[0]) / 'lib') + mpdecimal = glob.glob(str(self.prefix / 'opt/_internal/mpdecimal-*')) + if mpdecimal: + paths.append(Path(mpdecimal[0]) / 'lib') + object.__setattr__(self, 'library_path', paths) # Set excluded libraries. @@ -307,21 +312,56 @@ class ImageExtractor: '''Manylinux image tag.''' - def extract(self, destination: Path): + def default_destination(self): + return self.prefix / f'extracted/{self.tag}' + + + def extract(self, destination: Optional[Path]=None, *, cleanup=False): '''Extract Manylinux image.''' + if destination is None: + destination = self.default_destination() + + if cleanup: + def cleanup(destination): + shutil.rmtree(destination, ignore_errors=True) + atexit.register(cleanup, destination) + with open(self.prefix / f'tags/{self.tag}.json') as f: meta = json.load(f) layers = meta['layers'] - for layer in layers: - debug('EXTRACT', f'{layer}.tar.gz') + extracted = [] + extracted_file = destination / '.extracted' + if destination.exists(): + clean_destination = True + if extracted_file.exists(): + with extracted_file.open() as f: + extracted = f.read().split(os.linesep)[:-1] + for a, b in zip(layers, extracted): + if a != b: + break + else: + clean_destination = False + + if clean_destination: + shutil.rmtree(destination, ignore_errors=True) + + for i, layer in enumerate(layers): + try: + if layer == extracted[i]: + continue + except IndexError: + pass + + debug('EXTRACT', f'{layer}.tar.gz') filename = self.prefix / f'layers/{layer}.tar.gz' - cmd = ' && '.join(( - f'mkdir -p {destination}', - f'tar -xzf {filename} -C {destination}', - f'chmod u+rw -R {destination}' + cmd = ''.join(( + f'trap \'chmod u+rw -R {destination}\' EXIT ; ', + f'mkdir -p {destination} && ', + f'tar -xzf {filename} -C {destination} && ', + f'echo \'{layer}\' >> {extracted_file}' )) - process = subprocess.run(cmd, shell=True, check=True, - capture_output=True) + process = subprocess.run(f'/bin/bash -c "{cmd}"', shell=True, + check=True, capture_output=True) diff --git a/python_appimage/utils/deps.py b/python_appimage/utils/deps.py index 3fa3113..0469e80 100644 --- a/python_appimage/utils/deps.py +++ b/python_appimage/utils/deps.py @@ -11,22 +11,22 @@ from .url import urlretrieve _ARCH = platform.machine() -_CACHE_DIR = os.path.expanduser('~/.cache/python-appimage') - +CACHE_DIR = os.path.expanduser('~/.cache/python-appimage') +'''Package cache location''' PREFIX = os.path.abspath(os.path.dirname(__file__) + '/..') '''Package installation prefix''' -APPIMAGETOOL_DIR = os.path.join(_CACHE_DIR, 'bin') +APPIMAGETOOL_DIR = os.path.join(CACHE_DIR, 'bin') '''Location of the appimagetool binary''' APPIMAGETOOL_VERSION = '12' '''Version of the appimagetool binary''' -EXCLUDELIST = os.path.join(_CACHE_DIR, 'share/excludelist') +EXCLUDELIST = os.path.join(CACHE_DIR, 'share/excludelist') '''AppImage exclusion list''' -PATCHELF = os.path.join(_CACHE_DIR, 'bin/patchelf') +PATCHELF = os.path.join(CACHE_DIR, 'bin/patchelf') '''Location of the PatchELF binary''' PATCHELF_VERSION = '0.14.3' @@ -93,7 +93,7 @@ def ensure_patchelf(): if os.path.exists(PATCHELF): return False - tgz = '-'.join(('patchelf', _PATCHELF_VERSION, _ARCH)) + '.tar.gz' + tgz = '-'.join(('patchelf', PATCHELF_VERSION, _ARCH)) + '.tar.gz' baseurl = 'https://github.com/NixOS/patchelf' log('INSTALL', 'patchelf from %s', baseurl) @@ -102,7 +102,7 @@ def ensure_patchelf(): make_tree(dirname) with TemporaryDirectory() as tmpdir: urlretrieve(os.path.join(baseurl, 'releases', 'download', - _PATCHELF_VERSION, tgz), tgz) + PATCHELF_VERSION, tgz), tgz) system(('tar', 'xzf', tgz)) copy_file('bin/patchelf', patchelf) os.chmod(patchelf, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) diff --git a/python_appimage/utils/url.py b/python_appimage/utils/url.py index 0cdd9c8..5dc98fd 100644 --- a/python_appimage/utils/url.py +++ b/python_appimage/utils/url.py @@ -32,9 +32,9 @@ def urlretrieve(url, filename=None): else: debug('DOWNLOAD', '%s as %s', url, filename) - parent_directory = os.path.dirname(filename) - if not os.path.exists(parent_directory): - os.makedirs(parent_directory) + parent_directory = os.path.dirname(filename) + if parent_directory and not os.path.exists(parent_directory): + os.makedirs(parent_directory) if _urlretrieve is None: data = urllib2.urlopen(url).read()