From eb05b77a858d53aa9e97b17893c3766a2c9e6d45 Mon Sep 17 00:00:00 2001 From: Valentin Niess Date: Wed, 21 May 2025 21:23:30 +0200 Subject: [PATCH] Update the list command --- python_appimage/commands/build/manylinux.py | 21 +++----- python_appimage/commands/list.py | 38 +++++--------- python_appimage/manylinux/__init__.py | 24 ++++++++- python_appimage/manylinux/download.py | 5 +- python_appimage/manylinux/extract.py | 2 + python_appimage/utils/docker.py | 58 --------------------- python_appimage/utils/manylinux.py | 14 ----- scripts/test-appimage.py | 2 +- 8 files changed, 47 insertions(+), 117 deletions(-) delete mode 100644 python_appimage/utils/docker.py delete mode 100644 python_appimage/utils/manylinux.py diff --git a/python_appimage/commands/build/manylinux.py b/python_appimage/commands/build/manylinux.py index c655f94..7442b2d 100644 --- a/python_appimage/commands/build/manylinux.py +++ b/python_appimage/commands/build/manylinux.py @@ -3,8 +3,7 @@ from pathlib import Path import shutil from ...appimage import build_appimage -from ...manylinux import Arch, Downloader, ImageExtractor, LinuxTag, \ - PythonExtractor +from ...manylinux import ensure_image, PythonExtractor from ...utils.tmp import TemporaryDirectory @@ -21,21 +20,13 @@ def execute(tag, abi): '''Build a Python AppImage using a Manylinux image ''' - tag, arch = tag.split('_', 1) - tag = LinuxTag.from_brief(tag) - arch = Arch.from_str(arch) - - downloader = Downloader(tag=tag, arch=arch) - downloader.download() - - image_extractor = ImageExtractor(downloader.default_destination()) - image_extractor.extract() + image = ensure_image(tag) pwd = os.getcwd() with TemporaryDirectory() as tmpdir: python_extractor = PythonExtractor( - arch = arch, - prefix = image_extractor.default_destination(), + arch = image.arch, + prefix = image.path, tag = abi ) appdir = Path(tmpdir) / 'AppDir' @@ -44,7 +35,7 @@ def execute(tag, abi): fullname = '-'.join(( f'{python_extractor.impl}{python_extractor.version.long()}', abi, - f'{tag}_{arch}' + f'{image.tag}_{image.arch}' )) destination = f'{fullname}.AppImage' @@ -52,7 +43,7 @@ def execute(tag, abi): appdir = str(appdir), destination = destination ) - shutil.move( + shutil.copy( Path(tmpdir) / destination, Path(pwd) / destination ) diff --git a/python_appimage/commands/list.py b/python_appimage/commands/list.py index 083a4d2..6b3f526 100644 --- a/python_appimage/commands/list.py +++ b/python_appimage/commands/list.py @@ -1,8 +1,8 @@ -import os +import glob +from pathlib import Path -from ..utils.docker import docker_run +from ..manylinux import ensure_image, PythonVersion from ..utils.log import log -from ..utils.tmp import TemporaryDirectory __all__ = ['execute'] @@ -18,26 +18,16 @@ def execute(tag): '''List python versions installed in a manylinux image ''' - with TemporaryDirectory() as tmpdir: - script = ( - 'for dir in $(ls /opt/python | grep "^cp[0-9]"); do', - ' version=$(/opt/python/$dir/bin/python -c "import sys; ' \ - 'sys.stdout.write(sys.version.split()[0])")', - ' echo "$dir $version"', - 'done', - ) - if tag.startswith('2_'): - image = 'manylinux_' + tag - else: - image = 'manylinux' + tag - result = docker_run( - 'quay.io/pypa/' + image, - script, - capture = True - ) - pythons = [line.split() for line in result.split(os.linesep) if line] + image = ensure_image(tag) - for (abi, version) in pythons: - log('LIST', "{:7} -> /opt/python/{:}".format(version, abi)) + pythons = [] + for path in glob.glob(str(image.path / 'opt/python/cp*')): + path = Path(path) + version = PythonVersion.from_str(path.readlink().name[8:]).long() + pythons.append((path.name, version)) + pythons = sorted(pythons) - return pythons + for (abi, version) in pythons: + log('LIST', "{:8} -> /opt/python/{:}".format(version, abi)) + + return pythons diff --git a/python_appimage/manylinux/__init__.py b/python_appimage/manylinux/__init__.py index 7dcda3a..419caba 100644 --- a/python_appimage/manylinux/__init__.py +++ b/python_appimage/manylinux/__init__.py @@ -1,7 +1,29 @@ +from types import SimpleNamespace + from .config import Arch, LinuxTag, PythonImpl, PythonVersion from .download import Downloader from .extract import ImageExtractor, PythonExtractor -__all__ = ['Arch', 'Downloader', 'ImageExtractor', 'LinuxTag', +__all__ = ['Arch', 'Downloader', 'ensure_image', 'ImageExtractor', 'LinuxTag', 'PythonExtractor', 'PythonImpl', 'PythonVersion'] + + +def ensure_image(tag): + '''Extract a manylinux image to the cache''' + + tag, arch = tag.split('_', 1) + tag = LinuxTag.from_brief(tag) + arch = Arch.from_str(arch) + + downloader = Downloader(tag=tag, arch=arch) + downloader.download() + + image_extractor = ImageExtractor(downloader.default_destination()) + image_extractor.extract() + + return SimpleNamespace( + arch = arch, + tag = tag, + path = image_extractor.default_destination(), + ) diff --git a/python_appimage/manylinux/download.py b/python_appimage/manylinux/download.py index 33c798a..bfc2cf2 100644 --- a/python_appimage/manylinux/download.py +++ b/python_appimage/manylinux/download.py @@ -22,10 +22,6 @@ class DownloadError(Exception): pass -class TarError(Exception): - pass - - @dataclass(frozen=True) class Downloader: @@ -68,6 +64,7 @@ class Downloader: # Authenticate to quay.io. repository = f'pypa/{self.image}' + log('PULL', f'{self.image}:{tag}') url = 'https://quay.io/v2/auth' url = f'{url}?service=quay.io&scope=repository:{repository}:pull' debug('GET', url) diff --git a/python_appimage/manylinux/extract.py b/python_appimage/manylinux/extract.py index 4e29ff7..8ce301d 100644 --- a/python_appimage/manylinux/extract.py +++ b/python_appimage/manylinux/extract.py @@ -346,6 +346,8 @@ class ImageExtractor: shutil.rmtree(destination, ignore_errors=True) atexit.register(cleanup, destination) + log('EXTRACT', f'{self.prefix.name}:{self.tag}') + with open(self.prefix / f'tags/{self.tag}.json') as f: meta = json.load(f) layers = meta['layers'] diff --git a/python_appimage/utils/docker.py b/python_appimage/utils/docker.py deleted file mode 100644 index 1713191..0000000 --- a/python_appimage/utils/docker.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import platform -import stat -import subprocess -import sys - -from .compat import decode -from .log import log -from .system import system - - -def docker_run(image, extra_cmds, capture=False): - '''Execute commands within a docker container - ''' - - ARCH = platform.machine() - if image.endswith(ARCH): - bash_arg = '/pwd/run.sh' - elif image.endswith('i686') and ARCH == 'x86_64': - bash_arg = '-c "linux32 /pwd/run.sh"' - elif image.endswith('x86_64') and ARCH == 'i686': - bash_arg = '-c "linux64 /pwd/run.sh"' - else: - raise ValueError('Unsupported Docker image: ' + image) - - log('PULL', image) - system(('docker', 'pull', image)) - - script = [ - 'set -e', - 'trap "chown -R {:}:{:} *" EXIT'.format(os.getuid(), - os.getgid()), - 'cd /pwd' - ] - - script += extra_cmds - - with open('run.sh', 'w') as f: - f.write(os.linesep.join(script)) - os.chmod('run.sh', stat.S_IRWXU) - - cmd = ' '.join(('docker', 'run', '--mount', - 'type=bind,source={:},target=/pwd'.format(os.getcwd()), - image, '/bin/bash', bash_arg)) - - if capture: - opts = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE} - else: - opts = {} - log('RUN', image) - p = subprocess.Popen(cmd, shell=True, **opts) - r = p.communicate() - if p.returncode != 0: - if p.returncode == 139: - sys.stderr.write("segmentation fault when running Docker (139)\n") - sys.exit(p.returncode) - if capture: - return decode(r[0]) diff --git a/python_appimage/utils/manylinux.py b/python_appimage/utils/manylinux.py deleted file mode 100644 index 9b45882..0000000 --- a/python_appimage/utils/manylinux.py +++ /dev/null @@ -1,14 +0,0 @@ -def format_appimage_name(abi, version, tag): - '''Format the Python AppImage name using the ABI, python version and OS tags - ''' - return 'python{:}-{:}-{:}.AppImage'.format( - version, abi, format_tag(tag)) - - -def format_tag(tag): - '''Format Manylinux tag - ''' - if tag.startswith('2_'): - return 'manylinux_' + tag - else: - return 'manylinux' + tag diff --git a/scripts/test-appimage.py b/scripts/test-appimage.py index 13a1922..7ebaa35 100755 --- a/scripts/test-appimage.py +++ b/scripts/test-appimage.py @@ -162,7 +162,7 @@ assert_eq(expected, sys.prefix) # Test SSL (see issue #24). if version.major > 2: - Script(f''' + Script(''' from http import HTTPStatus import urllib.request with urllib.request.urlopen('https://wikipedia.org') as r: