diff --git a/python_appimage/appimage/__init__.py b/python_appimage/appimage/__init__.py index f976473..bab6c12 100644 --- a/python_appimage/appimage/__init__.py +++ b/python_appimage/appimage/__init__.py @@ -1,7 +1,7 @@ from .build import build_appimage -from .relocate import cert_file_env_string, patch_binary, relocate_python, \ - tcltk_env_string +from .appify import Appifier, tcltk_env_string +from .relocate import patch_binary, relocate_python -__all__ = ['build_appimage', 'cert_file_env_string', 'patch_binary', - 'relocate_python', 'tcltk_env_string'] +__all__ = ['Appifier', 'build_appimage', 'patch_binary', 'relocate_python', + 'tcltk_env_string'] diff --git a/python_appimage/appimage/appify.py b/python_appimage/appimage/appify.py new file mode 100644 index 0000000..817f56d --- /dev/null +++ b/python_appimage/appimage/appify.py @@ -0,0 +1,269 @@ +from dataclasses import dataclass +import glob +import os +import re +from typing import Optional, Tuple + +from ..utils.deps import PREFIX +from ..utils.fs import copy_file, copy_tree, make_tree, remove_file, remove_tree +from ..utils.log import debug, log +from ..utils.template import copy_template, load_template + + +@dataclass(frozen=True) +class Appifier: + + '''Path to AppDir root.''' + appdir: str + + '''Path to AppDir executables.''' + appdir_bin: str + + '''Path to Python executables.''' + python_bin: str + + '''Path to Python site-packages.''' + python_pkg: str + + '''Tcl/Tk version.''' + tk_version: str + + '''Python version.''' + version: 'PythonVersion' + + '''Path to SSL certification file.''' + cert_src: Optional[str]=None + + + def appify(self): + + python_x_y = f'python{self.version.short()}' + pip_x_y = f'pip{self.version.short()}' + + # Add a runtime patch for sys.executable, before site.main() execution + log('PATCH', f'{python_x_y} sys.executable') + set_executable_patch( + self.version.short(), + self.python_pkg, + PREFIX + '/data/_initappimage.py' + ) + + # Set a hook for cleaning sys.path, after site.main() execution + log('HOOK', f'{python_x_y} sys.path') + + sitepkgs = self.python_pkg + '/site-packages' + make_tree(sitepkgs) + copy_file(PREFIX + '/data/sitecustomize.py', sitepkgs) + + # Symlink SSL certificates + # (see https://github.com/niess/python-appimage/issues/24) + cert_file = '/opt/_internal/certs.pem' + cert_dst = f'{self.appdir}{cert_file}' + if self.cert_src is not None: + if os.path.exists(self.cert_src): + if not os.path.exists(cert_dst): + dirname, basename = os.path.split(cert_dst) + relpath = os.path.relpath(self.cert_src, dirname) + make_tree(dirname) + os.symlink(relpath, cert_dst) + log('INSTALL', basename) + if not os.path.exists(cert_dst): + cert_file = None + + # Bundle the python wrapper + wrapper = f'{self.appdir_bin}/{python_x_y}' + if not os.path.exists(wrapper): + log('INSTALL', f'{python_x_y} wrapper') + entrypoint_path = PREFIX + '/data/entrypoint.sh' + entrypoint = load_template( + entrypoint_path, + python=f'python{self.version.flavoured()}' + ) + dictionary = { + 'entrypoint': entrypoint, + 'shebang': '#! /bin/bash', + 'tcltk-env': tcltk_env_string(self.python_pkg, self.tk_version) + } + if cert_file: + dictionary['cert-file'] = cert_file_env_string(cert_file) + else: + dictionary['cert-file'] = '' + + _copy_template('python-wrapper.sh', wrapper, **dictionary) + + # Set or update symlinks to python and pip. + pip_target = f'{self.python_bin}/{pip_x_y}' + if os.path.exists(pip_target): + relpath = os.path.relpath(pip_target, self.appdir_bin) + os.symlink(relpath, f'{self.appdir_bin}/{pip_x_y}') + + pythons = glob.glob(self.appdir_bin + '/python?.*') + versions = [os.path.basename(python)[6:] for python in pythons] + latest2, latest3 = '0.0', '0.0' + for version in versions: + if version.startswith('2') and version >= latest2: + latest2 = version + elif version.startswith('3') and version >= latest3: + latest3 = version + if latest2 == self.version.short(): + python2 = self.appdir_bin + '/python2' + remove_file(python2) + os.symlink(python_x_y, python2) + has_pip = os.path.exists(self.appdir_bin + '/' + pip_x_y) + if has_pip: + pip2 = self.appdir_bin + '/pip2' + remove_file(pip2) + os.symlink(pip_x_y, pip2) + if latest3 == '0.0': + log('SYMLINK', 'python, python2 to ' + python_x_y) + python = self.appdir_bin + '/python' + remove_file(python) + os.symlink('python2', python) + if has_pip: + log('SYMLINK', 'pip, pip2 to ' + pip_x_y) + pip = self.appdir_bin + '/pip' + remove_file(pip) + os.symlink('pip2', pip) + else: + log('SYMLINK', 'python2 to ' + python_x_y) + if has_pip: + log('SYMLINK', 'pip2 to ' + pip_x_y) + elif latest3 == self.version.short(): + log('SYMLINK', 'python, python3 to ' + python_x_y) + python3 = self.appdir_bin + '/python3' + remove_file(python3) + os.symlink(python_x_y, python3) + python = self.appdir_bin + '/python' + remove_file(python) + os.symlink('python3', python) + if os.path.exists(self.appdir_bin + '/' + pip_x_y): + log('SYMLINK', 'pip, pip3 to ' + pip_x_y) + pip3 = self.appdir_bin + '/pip3' + remove_file(pip3) + os.symlink(pip_x_y, pip3) + pip = self.appdir_bin + '/pip' + remove_file(pip) + os.symlink('pip3', pip) + + # Bundle the entry point + apprun = f'{self.appdir}/AppRun' + if not os.path.exists(apprun): + log('INSTALL', 'AppRun') + + relpath = os.path.relpath(wrapper, self.appdir) + os.symlink(relpath, apprun) + + # Bundle the desktop file + desktop_name = f'python{self.version.long()}.desktop' + desktop = os.path.join(self.appdir, desktop_name) + if not os.path.exists(desktop): + log('INSTALL', desktop_name) + apps = 'usr/share/applications' + appfile = f'{self.appdir}/{apps}/{desktop_name}' + if not os.path.exists(appfile): + make_tree(os.path.join(self.appdir, apps)) + _copy_template('python.desktop', appfile, + version=self.version.short(), + fullversion=self.version.long()) + os.symlink(os.path.join(apps, desktop_name), desktop) + + # Bundle icons + icons = 'usr/share/icons/hicolor/256x256/apps' + icon = os.path.join(self.appdir, 'python.png') + if not os.path.exists(icon): + log('INSTALL', 'python.png') + make_tree(os.path.join(self.appdir, icons)) + copy_file(PREFIX + '/data/python.png', + os.path.join(self.appdir, icons, 'python.png')) + os.symlink(os.path.join(icons, 'python.png'), icon) + + diricon = os.path.join(self.appdir, '.DirIcon') + if not os.path.exists(diricon): + os.symlink('python.png', diricon) + + # Bundle metadata + meta_name = f'python{self.version.long()}.appdata.xml' + meta_dir = os.path.join(self.appdir, 'usr/share/metainfo') + meta_file = os.path.join(meta_dir, meta_name) + if not os.path.exists(meta_file): + log('INSTALL', meta_name) + make_tree(meta_dir) + _copy_template( + 'python.appdata.xml', + meta_file, + version = self.version.short(), + fullversion = self.version.long() + ) + + +def cert_file_env_string(cert_file): + '''Environment for using a bundled certificate + ''' + if cert_file: + return ''' +# Export SSL certificate +export SSL_CERT_FILE="${{APPDIR}}{cert_file:}"'''.format( + cert_file=cert_file) + else: + return '' + + +def _copy_template(name, destination, **kwargs): + path = os.path.join(PREFIX, 'data', name) + copy_template(path, destination, **kwargs) + + +def tcltk_env_string(python_pkg, tk_version): + '''Environment for using AppImage's TCl/Tk + ''' + + if tk_version: + return ''' +# Export TCl/Tk +export TCL_LIBRARY="${{APPDIR}}/usr/share/tcltk/tcl{tk_version:}" +export TK_LIBRARY="${{APPDIR}}/usr/share/tcltk/tk{tk_version:}" +export TKPATH="${{TK_LIBRARY}}"'''.format( + tk_version=tk_version) + else: + return '' + +def set_executable_patch(version, pkgpath, patch): + '''Set a runtime patch for sys.executable name + ''' + + # This patch needs to be executed before site.main() is called. A natural + # option is to apply it directy to the site module. But, starting with + # Python 3.11, the site module is frozen within Python executable. Then, + # doing so would require to recompile Python. Thus, starting with 3.11 we + # instead apply the patch to the encodings package. Indeed, the latter is + # loaded before the site module, and it is not frozen (as for now). + major, minor = [int(v) for v in version.split('.')] + if (major >= 3) and (minor >= 11): + path = os.path.join(pkgpath, 'encodings', '__init__.py') + else: + path = os.path.join(pkgpath, 'site.py') + + with open(path) as f: + source = f.read() + + if '_initappimage' in source: return + + lines = source.split(os.linesep) + + if path.endswith('site.py'): + # Insert the patch before the main function + for i, line in enumerate(lines): + if line.startswith('def main('): break + else: + # Append the patch at end of file + i = len(lines) + + with open(patch) as f: + patch = f.read() + + lines.insert(i, patch) + lines.insert(i + 1, '') + + source = os.linesep.join(lines) + with open(path, 'w') as f: + f.write(source) diff --git a/python_appimage/appimage/build.py b/python_appimage/appimage/build.py index 2538125..7cc4f0a 100644 --- a/python_appimage/appimage/build.py +++ b/python_appimage/appimage/build.py @@ -21,7 +21,7 @@ def build_appimage(appdir=None, destination=None): if appdir is None: appdir = 'AppDir' - log('BUILD', appdir) + log('BUILD', os.path.basename(appdir)) appimagetool = ensure_appimagetool() arch = platform.machine() diff --git a/python_appimage/appimage/relocate.py b/python_appimage/appimage/relocate.py index 2914c92..9521296 100644 --- a/python_appimage/appimage/relocate.py +++ b/python_appimage/appimage/relocate.py @@ -4,71 +4,16 @@ import re import shutil import sys +from .appify import Appifier +from ..manylinux import PythonVersion from ..utils.deps import EXCLUDELIST, PATCHELF, PREFIX, ensure_excludelist, \ ensure_patchelf from ..utils.fs import copy_file, copy_tree, make_tree, remove_file, remove_tree from ..utils.log import debug, log from ..utils.system import ldd, system -from ..utils.template import copy_template, load_template -__all__ = ["cert_file_env_string", "patch_binary", "relocate_python", - "tcltk_env_string"] - - -def _copy_template(name, destination, **kwargs): - path = os.path.join(PREFIX, 'data', name) - copy_template(path, destination, **kwargs) - - -def _get_tk_version(python_pkg): - tkinter = glob.glob(python_pkg + '/lib-dynload/_tkinter*.so') - if tkinter: - tkinter = tkinter[0] - for dep in ldd(tkinter): - name = os.path.basename(dep) - if name.startswith('libtk'): - match = re.search('libtk([0-9]+[.][0-9]+)', name) - return match.group(1) - else: - raise RuntimeError('could not guess Tcl/Tk version') - - -def _get_tk_libdir(version): - try: - library = system(('tclsh' + version,), stdin='puts [info library]') - except SystemError: - raise RuntimeError('could not locate Tcl/Tk' + version + ' library') - - return os.path.dirname(library) - - -def tcltk_env_string(python_pkg): - '''Environment for using AppImage's TCl/Tk - ''' - tk_version = _get_tk_version(python_pkg) - - if tk_version: - return ''' -# Export TCl/Tk -export TCL_LIBRARY="${{APPDIR}}/usr/share/tcltk/tcl{tk_version:}" -export TK_LIBRARY="${{APPDIR}}/usr/share/tcltk/tk{tk_version:}" -export TKPATH="${{TK_LIBRARY}}"'''.format( - tk_version=tk_version) - else: - return '' - - -def cert_file_env_string(cert_file): - '''Environment for using a bundled certificate - ''' - if cert_file: - return ''' -# Export SSL certificate -export SSL_CERT_FILE="${{APPDIR}}{cert_file:}"'''.format( - cert_file=cert_file) - else: - return '' +__all__ = ['patch_binary', 'relocate_python'] _excluded_libs = None @@ -116,48 +61,6 @@ def patch_binary(path, libdir, recursive=True): patch_binary(target, libdir, recursive=True) -def set_executable_patch(version, pkgpath, patch): - '''Set a runtime patch for sys.executable name - ''' - - # This patch needs to be executed before site.main() is called. A natural - # option is to apply it directy to the site module. But, starting with - # Python 3.11, the site module is frozen within Python executable. Then, - # doing so would require to recompile Python. Thus, starting with 3.11 we - # instead apply the patch to the encodings package. Indeed, the latter is - # loaded before the site module, and it is not frozen (as for now). - major, minor = [int(v) for v in version.split('.')] - if (major >= 3) and (minor >= 11): - path = os.path.join(pkgpath, 'encodings', '__init__.py') - else: - path = os.path.join(pkgpath, 'site.py') - - with open(path) as f: - source = f.read() - - if '_initappimage' in source: return - - lines = source.split(os.linesep) - - if path.endswith('site.py'): - # Insert the patch before the main function - for i, line in enumerate(lines): - if line.startswith('def main('): break - else: - # Append the patch at end of file - i = len(lines) - - with open(patch) as f: - patch = f.read() - - lines.insert(i, patch) - lines.insert(i + 1, '') - - source = os.linesep.join(lines) - with open(path, 'w') as f: - f.write(source) - - def relocate_python(python=None, appdir=None): '''Bundle a Python install inside an AppDir ''' @@ -255,9 +158,6 @@ def relocate_python(python=None, appdir=None): f.write(body) shutil.copymode(pip_source, target) - relpath = os.path.relpath(target, APPDIR_BIN) - os.symlink(relpath, APPDIR_BIN + '/' + PIP_X_Y) - # Remove unrelevant files log('PRUNE', '%s packages', PYTHON_X_Y) @@ -269,17 +169,6 @@ def relocate_python(python=None, appdir=None): for path in matches: remove_tree(path) - # Add a runtime patch for sys.executable, before site.main() execution - log('PATCH', '%s sys.executable', PYTHON_X_Y) - set_executable_patch(VERSION, PYTHON_PKG, PREFIX + '/data/_initappimage.py') - - # Set a hook for cleaning sys.path, after site.main() execution - log('HOOK', '%s sys.path', PYTHON_X_Y) - - sitepkgs = PYTHON_PKG + '/site-packages' - make_tree(sitepkgs) - copy_file(PREFIX + '/data/sitecustomize.py', sitepkgs) - # Set RPATHs and bundle external libraries log('LINK', '%s C-extensions', PYTHON_X_Y) @@ -320,111 +209,35 @@ def relocate_python(python=None, appdir=None): copy_file(cert_file, 'AppDir' + cert_file) log('INSTALL', basename) + # Bundle AppImage specific files. + appifier = Appifier( + appdir = APPDIR, + appdir_bin = APPDIR_BIN, + python_bin = PYTHON_BIN, + python_pkg = PYTHON_PKG, + tk_version = tk_version, + version = PythonVersion.from_str(FULLVERSION) + ) + appifier.appify() - # Bundle the python wrapper - wrapper = APPDIR_BIN + '/' + PYTHON_X_Y - if not os.path.exists(wrapper): - log('INSTALL', '%s wrapper', PYTHON_X_Y) - entrypoint_path = PREFIX + '/data/entrypoint.sh' - entrypoint = load_template(entrypoint_path, python=PYTHON_X_Y) - dictionary = {'entrypoint': entrypoint, - 'shebang': '#! /bin/bash', - 'tcltk-env': tcltk_env_string(PYTHON_PKG), - 'cert-file': cert_file_env_string(cert_file)} - _copy_template('python-wrapper.sh', wrapper, **dictionary) - # Set or update symlinks to python - pythons = glob.glob(APPDIR_BIN + '/python?.*') - versions = [os.path.basename(python)[6:] for python in pythons] - latest2, latest3 = '0.0', '0.0' - for version in versions: - if version.startswith('2') and version >= latest2: - latest2 = version - elif version.startswith('3') and version >= latest3: - latest3 = version - if latest2 == VERSION: - python2 = APPDIR_BIN + '/python2' - remove_file(python2) - os.symlink(PYTHON_X_Y, python2) - has_pip = os.path.exists(APPDIR_BIN + '/' + PIP_X_Y) - if has_pip: - pip2 = APPDIR_BIN + '/pip2' - remove_file(pip2) - os.symlink(PIP_X_Y, pip2) - if latest3 == '0.0': - log('SYMLINK', 'python, python2 to ' + PYTHON_X_Y) - python = APPDIR_BIN + '/python' - remove_file(python) - os.symlink('python2', python) - if has_pip: - log('SYMLINK', 'pip, pip2 to ' + PIP_X_Y) - pip = APPDIR_BIN + '/pip' - remove_file(pip) - os.symlink('pip2', pip) +def _get_tk_version(python_pkg): + tkinter = glob.glob(python_pkg + '/lib-dynload/_tkinter*.so') + if tkinter: + tkinter = tkinter[0] + for dep in ldd(tkinter): + name = os.path.basename(dep) + if name.startswith('libtk'): + match = re.search('libtk([0-9]+[.][0-9]+)', name) + return match.group(1) else: - log('SYMLINK', 'python2 to ' + PYTHON_X_Y) - if has_pip: - log('SYMLINK', 'pip2 to ' + PIP_X_Y) - elif latest3 == VERSION: - log('SYMLINK', 'python, python3 to ' + PYTHON_X_Y) - python3 = APPDIR_BIN + '/python3' - remove_file(python3) - os.symlink(PYTHON_X_Y, python3) - python = APPDIR_BIN + '/python' - remove_file(python) - os.symlink('python3', python) - if os.path.exists(APPDIR_BIN + '/' + PIP_X_Y): - log('SYMLINK', 'pip, pip3 to ' + PIP_X_Y) - pip3 = APPDIR_BIN + '/pip3' - remove_file(pip3) - os.symlink(PIP_X_Y, pip3) - pip = APPDIR_BIN + '/pip' - remove_file(pip) - os.symlink('pip3', pip) - - # Bundle the entry point - apprun = APPDIR + '/AppRun' - if not os.path.exists(apprun): - log('INSTALL', 'AppRun') - - relpath = os.path.relpath(wrapper, APPDIR) - os.symlink(relpath, APPDIR + '/AppRun') - - # Bundle the desktop file - desktop_name = 'python{:}.desktop'.format(FULLVERSION) - desktop = os.path.join(APPDIR, desktop_name) - if not os.path.exists(desktop): - log('INSTALL', desktop_name) - apps = 'usr/share/applications' - appfile = '{:}/{:}/python{:}.desktop'.format(APPDIR, apps, FULLVERSION) - if not os.path.exists(appfile): - make_tree(os.path.join(APPDIR, apps)) - _copy_template('python.desktop', appfile, version=VERSION, - fullversion=FULLVERSION) - os.symlink(os.path.join(apps, desktop_name), desktop) + raise RuntimeError('could not guess Tcl/Tk version') - # Bundle icons - icons = 'usr/share/icons/hicolor/256x256/apps' - icon = os.path.join(APPDIR, 'python.png') - if not os.path.exists(icon): - log('INSTALL', 'python.png') - make_tree(os.path.join(APPDIR, icons)) - copy_file(PREFIX + '/data/python.png', - os.path.join(APPDIR, icons, 'python.png')) - os.symlink(os.path.join(icons, 'python.png'), icon) +def _get_tk_libdir(version): + try: + library = system(('tclsh' + version,), stdin='puts [info library]') + except SystemError: + raise RuntimeError('could not locate Tcl/Tk' + version + ' library') - diricon = os.path.join(APPDIR, '.DirIcon') - if not os.path.exists(diricon): - os.symlink('python.png', diricon) - - - # Bundle metadata - meta_name = 'python{:}.appdata.xml'.format(FULLVERSION) - meta_dir = os.path.join(APPDIR, 'usr/share/metainfo') - meta_file = os.path.join(meta_dir, meta_name) - if not os.path.exists(meta_file): - log('INSTALL', meta_name) - make_tree(meta_dir) - _copy_template('python.appdata.xml', meta_file, version=VERSION, - fullversion=FULLVERSION) + return os.path.dirname(library) diff --git a/python_appimage/commands/build/manylinux.py b/python_appimage/commands/build/manylinux.py index 3923651..3245c3e 100644 --- a/python_appimage/commands/build/manylinux.py +++ b/python_appimage/commands/build/manylinux.py @@ -56,13 +56,20 @@ def execute(tag, abi): tag = abi ) appdir = Path(tmpdir) / 'AppDir' - python_extractor.extract(appdir) + python_extractor.extract(appdir, appify=True) fullname = '-'.join(( f'{python_extractor.impl}{python_extractor.version.long()}', abi, f'{tag}_{arch}' )) - shutil.move(appdir, os.path.join(pwd, fullname)) - # XXX build_appimage(destination=_get_appimage_name(abi, tag)) + destination = f'{fullname}.AppImage' + build_appimage( + appdir = str(appdir), + destination = destination + ) + shutil.move( + Path(tmpdir) / destination, + Path(pwd) / destination + ) diff --git a/python_appimage/manylinux/extract.py b/python_appimage/manylinux/extract.py index 3dd4a84..bb55de7 100644 --- a/python_appimage/manylinux/extract.py +++ b/python_appimage/manylinux/extract.py @@ -12,6 +12,7 @@ import subprocess from typing import Dict, List, NamedTuple, Optional, Union from .config import Arch, PythonImpl, PythonVersion +from ..appimage import Appifier from ..utils.deps import ensure_excludelist, ensure_patchelf, EXCLUDELIST, \ PATCHELF from ..utils.log import debug, log @@ -117,18 +118,20 @@ class PythonExtractor: self, destination: Path, *, + appify: Optional[bool]=False, python_prefix: Optional[str]=None, - system_prefix: Optional[str]=None + system_prefix: Optional[str]=None, ): '''Extract Python runtime.''' python = f'python{self.version.short()}' - runtime = f'bin/{python}' - packages = f'lib/python{self.version.flavoured()}' + flavoured_python = f'python{self.version.flavoured()}' + runtime = f'bin/{flavoured_python}' + packages = f'lib/{flavoured_python}' pip = f'bin/pip{self.version.short()}' if python_prefix is None: - python_prefix = f'opt/python{self.version.flavoured()}' + python_prefix = f'opt/{flavoured_python}' if system_prefix is None: system_prefix = 'usr' @@ -152,7 +155,7 @@ class PythonExtractor: short = Path(python_dest / f'bin/python{self.version.major}') short.unlink(missing_ok=True) - short.symlink_to(python) + short.symlink_to(flavoured_python) short = Path(python_dest / 'bin/python') short.unlink(missing_ok=True) short.symlink_to(f'python{self.version.major}') @@ -166,7 +169,7 @@ class PythonExtractor: f.write('#! /bin/sh\n') f.write(' '.join(( '"exec"', - f'"$(dirname $(readlink -f ${0}))/{python}"', + f'"$(dirname $(readlink -f ${0}))/{flavoured_python}"', '"$0"', '"$@"\n' ))) @@ -198,7 +201,7 @@ class PythonExtractor: (root / f).unlink() # Map binary dependencies. - libs = self.ldd(self.python_prefix / f'bin/{python}') + libs = self.ldd(self.python_prefix / f'bin/{flavoured_python}') path = Path(self.python_prefix / f'{packages}/lib-dynload') for module in glob.glob(str(path / "*.so")): l = self.ldd(module) @@ -249,6 +252,9 @@ class PythonExtractor: dst = python_dest / f'{packages}/site-packages/{src.name}' if not dst.exists(): shutil.copytree(src, dst, symlinks=True) + + cert_src = dst / 'cacert.pem' + assert(cert_src.exists()) else: raise NotImplementedError() @@ -272,6 +278,18 @@ class PythonExtractor: dst = tcltk_dir / name shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True) + if appify: + appifier = Appifier( + appdir = str(destination), + appdir_bin = str(system_dest / 'bin'), + python_bin = str(python_dest / 'bin'), + python_pkg = str(python_dest / packages), + version = self.version, + tk_version = tx_version, + cert_src = cert_src + ) + appifier.appify() + def ldd(self, target: Path) -> Dict[str, Path]: '''Cross-platform implementation of ldd, using readelf.'''