diff --git a/python_appimage/__main__.py b/python_appimage/__main__.py index a6491ca..fa45fc8 100644 --- a/python_appimage/__main__.py +++ b/python_appimage/__main__.py @@ -59,9 +59,6 @@ def main(): build_manylinux_parser.add_argument('abi', help='python ABI (e.g. cp37-cp37m)') - build_manylinux_parser.add_argument('--contained', help=argparse.SUPPRESS, - action='store_true', default=False) - build_app_parser = build_subparsers.add_parser('app', description='Build a Python application using a base AppImage') build_app_parser.add_argument('appdir', diff --git a/python_appimage/commands/build/manylinux.py b/python_appimage/commands/build/manylinux.py index bbd1e93..3923651 100644 --- a/python_appimage/commands/build/manylinux.py +++ b/python_appimage/commands/build/manylinux.py @@ -1,10 +1,13 @@ import glob import os +from pathlib import Path import platform import shutil import sys from ...appimage import build_appimage, relocate_python +from ...manylinux import Arch, Downloader, ImageExtractor, LinuxTag, \ + PythonExtractor from ...utils.docker import docker_run from ...utils.fs import copy_tree from ...utils.manylinux import format_appimage_name, format_tag @@ -17,7 +20,7 @@ __all__ = ['execute'] def _unpack_args(args): '''Unpack command line arguments ''' - return args.tag, args.abi, args.contained + return args.tag, args.abi def _get_appimage_name(abi, tag): @@ -31,74 +34,35 @@ def _get_appimage_name(abi, tag): return format_appimage_name(abi, fullversion, tag) -def execute(tag, abi, contained=False): - '''Build a Python AppImage using a manylinux docker image +def execute(tag, abi): + '''Build a Python AppImage using a Manylinux image ''' - if not contained: - # Forward the build to a Docker image - image = 'quay.io/pypa/' + format_tag(tag) - python = '/opt/python/' + abi + '/bin/python' + tag, arch = tag.split('_', 1) + tag = LinuxTag.from_brief(tag) + arch = Arch.from_str(arch) - pwd = os.getcwd() - dirname = os.path.abspath(os.path.dirname(__file__) + '/../..') - with TemporaryDirectory() as tmpdir: - copy_tree(dirname, 'python_appimage') + downloader = Downloader(tag=tag, arch=arch) + downloader.download() - argv = sys.argv[1:] - if argv: - argv = ' '.join(argv) - else: - argv = 'build manylinux {:} {:}'.format(tag, abi) - if tag.startswith("1_"): - # On manylinux1 tk is not installed - script = [ - 'yum --disablerepo="*" --enablerepo=base install -q -y tk'] - else: - # tk is already installed on other platforms - script = [] - script += [ - python + ' -m python_appimage ' + argv + ' --contained', - '' - ] - docker_run(image, script) + image_extractor = ImageExtractor(downloader.default_destination()) + image_extractor.extract() - appimage_name = _get_appimage_name(abi, tag) + pwd = os.getcwd() + with TemporaryDirectory() as tmpdir: + python_extractor = PythonExtractor( + arch = arch, + prefix = image_extractor.default_destination(), + tag = abi + ) + appdir = Path(tmpdir) / 'AppDir' + python_extractor.extract(appdir) - if tag.startswith('1_') or tag.startswith('2010_'): - # appimagetool does not run on manylinux1 (CentOS 5) or - # manylinux2010 (CentOS 6). Below is a patch for these specific - # cases. - arch = tag.split('_', 1)[-1] - if arch == platform.machine(): - # Pack the image directly from the host - build_appimage(destination=appimage_name) - else: - # Use a manylinux2014 Docker image (CentOS 7) in order to - # pack the image. - script = ( - 'python -m python_appimage ' + argv + ' --contained', - '' - ) - docker_run('quay.io/pypa/manylinux2014_' + arch, script) + fullname = '-'.join(( + f'{python_extractor.impl}{python_extractor.version.long()}', + abi, + f'{tag}_{arch}' + )) + shutil.move(appdir, os.path.join(pwd, fullname)) - shutil.move(appimage_name, os.path.join(pwd, appimage_name)) - - else: - # We are running within a manylinux Docker image - is_manylinux_old = tag.startswith('1_') or tag.startswith('2010_') - - if not os.path.exists('AppDir'): - # Relocate the targeted manylinux Python installation - relocate_python() - else: - # This is a second stage build. The Docker image has actually been - # overriden (see above). - is_manylinux_old = False - - if is_manylinux_old: - # Build only the AppDir when running within a manylinux1 Docker - # image because appimagetool does not support CentOS 5 or CentOS 6. - pass - else: - build_appimage(destination=_get_appimage_name(abi, tag)) + # XXX build_appimage(destination=_get_appimage_name(abi, tag)) diff --git a/python_appimage/manylinux/config.py b/python_appimage/manylinux/config.py index beafba1..d7cb24e 100644 --- a/python_appimage/manylinux/config.py +++ b/python_appimage/manylinux/config.py @@ -1,6 +1,6 @@ from enum import auto, Enum import platform -from typing import NamedTuple, Union +from typing import NamedTuple, Optional, Union __all__ = ['Arch', 'PythonImpl', 'PythonVersion'] @@ -52,11 +52,21 @@ class LinuxTag(Enum): else: raise NotImplementedError(value) + @classmethod + def from_brief(cls, value) -> 'LinuxTag': + if value.startswith('2_'): + return cls.from_str('manylinux_' + value) + else: + return cls.from_str('manylinux' + value) + class PythonImpl(Enum): '''Supported Python implementations.''' CPYTHON = auto() + def __str__(self): + return 'python' + class PythonVersion(NamedTuple): '''''' @@ -64,15 +74,33 @@ class PythonVersion(NamedTuple): major: int minor: int patch: Union[int, str] + flavour: Optional[str]=None @classmethod def from_str(cls, value: str) -> 'PythonVersion': major, minor, patch = value.split('.', 2) + try: + patch, flavour = patch.split('-', 1) + except ValueError: + flavour = None + else: + if flavour == 'nogil': + flavour = 't' + elif flavour == 'ucs2': + flavour = 'm' + elif flavour == 'ucs4': + flavour = 'mu' + else: + raise NotImplementedError(value) try: patch = int(patch) except ValueError: pass - return cls(int(major), int(minor), patch) + return cls(int(major), int(minor), patch, flavour) + + def flavoured(self) -> str: + flavour = self.flavour if self.flavour == 't' else '' + return f'{self.major}.{self.minor}{flavour}' def long(self) -> str: return f'{self.major}.{self.minor}.{self.patch}' diff --git a/python_appimage/manylinux/extract.py b/python_appimage/manylinux/extract.py index 26bb7cf..3dd4a84 100644 --- a/python_appimage/manylinux/extract.py +++ b/python_appimage/manylinux/extract.py @@ -12,7 +12,8 @@ import subprocess from typing import Dict, List, NamedTuple, Optional, Union from .config import Arch, PythonImpl, PythonVersion -from ..utils.deps import ensure_excludelist, EXCLUDELIST +from ..utils.deps import ensure_excludelist, ensure_patchelf, EXCLUDELIST, \ + PATCHELF from ..utils.log import debug, log @@ -106,17 +107,8 @@ class PythonExtractor: # Set patchelf, if not provided. if self.patchelf is None: - paths = ( - Path(__file__).parent / 'bin', - Path.home() / '.local/bin' - ) - for path in paths: - patchelf = path / 'patchelf' - if patchelf.exists(): - break - else: - raise NotImplementedError() - object.__setattr__(self, 'patchelf', patchelf) + ensure_patchelf() + object.__setattr__(self, 'patchelf', PATCHELF) else: assert(self.patchelf.exists()) @@ -124,6 +116,7 @@ class PythonExtractor: def extract( self, destination: Path, + *, python_prefix: Optional[str]=None, system_prefix: Optional[str]=None ): @@ -131,11 +124,11 @@ class PythonExtractor: python = f'python{self.version.short()}' runtime = f'bin/{python}' - packages = f'lib/{python}' + packages = f'lib/python{self.version.flavoured()}' pip = f'bin/pip{self.version.short()}' if python_prefix is None: - python_prefix = f'opt/{python}' + python_prefix = f'opt/python{self.version.flavoured()}' if system_prefix is None: system_prefix = 'usr' @@ -152,6 +145,8 @@ class PythonExtractor: raise NotImplementedError() # Clone Python runtime. + log('CLONE', + f'{python} from {self.python_prefix.relative_to(self.prefix)}') (python_dest / 'bin').mkdir(exist_ok=True, parents=True) shutil.copy(self.python_prefix / runtime, python_dest / runtime) @@ -191,6 +186,7 @@ class PythonExtractor: symlinks=True, dirs_exist_ok=True) # Remove some clutters. + log('PRUNE', '%s packages', python) shutil.rmtree(python_dest / packages / 'test', ignore_errors=True) for root, dirs, files in os.walk(python_dest / packages): root = Path(root) @@ -226,6 +222,7 @@ class PythonExtractor: self.set_rpath(dst, '$ORIGIN') # Patch RPATHs of binary modules. + log('LINK', '%s C-extensions', python) path = Path(python_dest / f'{packages}/lib-dynload') for module in glob.glob(str(path / "*.so")): src = Path(module) @@ -245,6 +242,7 @@ class PythonExtractor: assert(certifi.name == 'certifi') site_packages = certifi.parent assert(site_packages.name == 'site-packages') + log('INSTALL', certifi.name) for src in glob.glob(str(site_packages / 'certifi*')): src = Path(src) @@ -264,6 +262,7 @@ class PythonExtractor: tx_version.sort() tx_version = tx_version[-1] + log('INSTALL', f'Tcl/Tk{tx_version}') tcltk_dir = Path(system_dest / 'share/tcltk') tcltk_dir.mkdir(exist_ok=True, parents=True)