34 Commits

Author SHA1 Message Date
Valentin Niess
573b4742d2 Patch openssl path detection 2025-07-02 12:48:39 +02:00
Valentin Niess
b870853d69 Setuptools dependency 2025-07-02 12:05:38 +02:00
Valentin Niess
fa25a18ba8 Smarter cacert.pem detection 2025-07-02 11:17:30 +02:00
Valentin Niess
a072354789 Bump version: v1.4.4 2025-05-24 21:18:17 +02:00
Valentin Niess
0376d42eca Use appimagetool continuous build 2025-05-24 21:12:55 +02:00
Valentin Niess
04071b3df9 Pin pypa/gh-action-pypi-publish@release/v1 2025-05-23 18:49:31 +02:00
Valentin Niess
f049580e2b Bump version: v1.4.3 2025-05-23 18:44:33 +02:00
Valentin Niess
0fabfe2810 Bundle version.py file 2025-05-23 18:28:34 +02:00
Valentin Niess
c7df0f9267 Bump version: v1.4.2 2025-05-23 18:25:24 +02:00
Valentin Niess
ab7eac67d5 Switch to pyproject.toml 2025-05-23 18:24:43 +02:00
Valentin Niess
8664b711d8 Bump version: v1.4.1 2025-05-23 16:24:39 +02:00
Valentin Niess
ad647e0ece Update workflows' dependencies 2025-05-23 16:19:34 +02:00
Valentin Niess
ac7df52db8 Bump version: v1.4.0 2025-05-23 15:46:05 +02:00
Valentin Niess
14da50382d Update requirements (Python 3.9, requests) 2025-05-23 15:34:27 +02:00
Valentin Niess
839cbc3fd4 Clean the documentation 2025-05-23 15:15:17 +02:00
Valentin Niess
19fe0dbab9 Patch the docs table 2025-05-23 10:32:53 +02:00
Valentin Niess
f9b46b5e7f Add aarch64 to the update 2025-05-22 23:46:41 +02:00
Valentin Niess
d1eb24e0f4 Manylinux1 patch 2025-05-22 17:43:43 +02:00
Valentin Niess
f6dd10d6b7 Skip some tests 2025-05-22 15:41:10 +02:00
Valentin Niess
39d91e847f No packaging option 2025-05-22 13:36:30 +02:00
Valentin Niess
ca8d4717d4 Forward arch when building 2025-05-22 12:10:33 +02:00
Valentin Niess
954e9bc91a Log tests results 2025-05-22 11:11:17 +02:00
Valentin Niess
5cf73a59d6 Build bare tarball 2025-05-22 09:55:15 +02:00
Valentin Niess
f746e5dae3 Get images to the cache 2025-05-22 09:32:51 +02:00
Valentin Niess
a7c56e3c77 Build specific image tags 2025-05-22 09:04:09 +02:00
Valentin Niess
2f663acc32 Cache management 2025-05-21 23:04:04 +02:00
Valentin Niess
eb05b77a85 Update the list command 2025-05-21 21:28:12 +02:00
Valentin Niess
fa3ec89228 Appimage test script 2025-05-21 20:27:51 +02:00
Valentin Niess
60aec8ba25 Patch some mistakes (according to flake8) 2025-05-20 23:27:20 +02:00
Valentin Niess
331fc6ab7f Relocate appification 2025-05-20 22:43:57 +02:00
Valentin Niess
75f10bbdc4 Update manylinux CLI 2025-05-20 16:16:27 +02:00
Valentin Niess
5fe8c22eb8 Harmonise installation prefixes 2025-05-20 08:52:56 +02:00
Valentin Niess
8090602b0f Tweak manylinux image extractor 2025-05-19 22:37:45 +02:00
Valentin Niess
1fdb439e70 Manylinux downloader and extractor 2025-05-19 17:13:29 +02:00
41 changed files with 2012 additions and 758 deletions

View File

@@ -23,7 +23,7 @@ jobs:
- uses: actions/checkout@v4
- name: Install Dependencies
run: pip install PyGithub
run: pip install PyGithub requests
- name: Run updater
run: |

View File

@@ -21,17 +21,20 @@ on:
jobs:
Test:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
strategy:
matrix:
version: ['2.7', '3.7', '3.9']
version: ['3.9']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.version }}
- name: Install Dependencies
run: pip install requests
- name: Test scipy
if: ${{ inputs.scipy }}
run: |
@@ -57,8 +60,9 @@ jobs:
./xonsh-x86_64.AppImage -c 'import xonsh'
- name: Test ssh-mitm
if: ${{ inputs.ssh_mitm && (matrix.version == '3.9') }}
if: ${{ inputs.ssh-mitm }}
run: |
python -m python_appimage build app applications/ssh-mitm
python -m python_appimage build app applications/ssh-mitm \
--python-version=3.11
test -e ssh-mitm-x86_64.AppImage
./ssh-mitm-x86_64.AppImage --help

View File

@@ -1,15 +1,11 @@
name: PyPI
on:
push:
branches:
- master
paths:
- 'VERSION'
workflow_dispatch:
inputs:
upload:
description: 'Upload to PyPI'
required: true
default: false
type: boolean
jobs:
@@ -25,6 +21,9 @@ jobs:
with:
python-version: ${{ matrix.version }}
- name: Install Dependencies
run: pip install requests
- name: Test local builder
run: |
python -m python_appimage build local -p $(which python) \
@@ -44,12 +43,11 @@ jobs:
- name: Build wheel
run: |
pip install -U pip
pip install -U wheel
python setup.py bdist_wheel --universal
pip install -U pip build
python -m build
- name: Upload to PyPI
if: (github.ref == 'refs/heads/master') && inputs.upload
uses: pypa/gh-action-pypi-publish@master
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_TOKEN }}

4
.gitignore vendored
View File

@@ -5,8 +5,6 @@ __pycache__
AppDir
build/*
dist
docs/ENV
python_appimage.egg-info
python_appimage/bin
python_appimage/data/excludelist
python_appimage/version.py
!python_appimage/commands/build

View File

@@ -1,9 +1,13 @@
version: 2
build:
os: ubuntu-22.04
tools:
python: "3.9"
mkdocs:
configuration: docs/mkdocs.yml
python:
version: 3.8
install:
- requirements: docs/requirements.txt
install:
- requirements: docs/requirements.txt

1
MANIFEST.in Normal file
View File

@@ -0,0 +1 @@
include python_appimage/data/*

View File

@@ -5,18 +5,18 @@ _Ready to use AppImages of Python are available as GitHub [releases][RELEASES]._
## Quickstart
We provide relocatable Python runtimes as [AppImages][APPIMAGE] for Linux
systems. These runtimes are extracted from [Manylinux][MANYLINUX] Docker images,
and they are available as GitHub [releases][RELEASES]. Our Python AppImages are
updated weekly, on every Sunday.
We provide relocatable Python runtimes in the form of [AppImages][APPIMAGE] for
Linux systems. These runtimes are extracted from [Manylinux][MANYLINUX] Docker
images and are available as GitHub [releases][RELEASES]. Our Python AppImages
are updated weekly, on every Sunday.
Instructions for _installing_ and running _Python AppImages_ are provided on
Instructions for _installing_ and running _Python AppImages_ can be found on
[Read the Docs][READTHEDOCS].
In addition, the online documentation describes the [`python-appimage`][PYPI]
utility, for application developers. This utility can facilitate the building of
Python apps, given an existing Python AppImage and a recipe folder.
[Examples][APPLICATIONS] of recipes are available from GitHub.
The online documentation also describes the [`python-appimage`][PYPI] utility
for application developers. This utility can facilitate the development of
Python applications, provided you have an existing Python AppImage and a recipe
folder. [Examples][APPLICATIONS] of recipes are available on GitHub.
## Projects using [`python-appimage`][GITHUB]

View File

@@ -1 +0,0 @@
1.3.1

View File

@@ -12,7 +12,9 @@
[GITHUB]: {{ config.repo_url }}
[LINUXDEPLOY]: https://github.com/linuxdeploy/linuxdeploy/
[MANYLINUX]: https://github.com/pypa/manylinux/
[NUMPY]: https://numpy.org/
[PATCHELF]: https://github.com/NixOS/patchelf/
[PEP_425]: https://peps.python.org/pep-0425/
[PYPI]: https://pypi.org/project/python-appimage/
[RELEASES]: {{ config.repo_url }}releases/
[SHEBANG]: https://en.wikipedia.org/wiki/Shebang_(Unix)/

View File

@@ -10,7 +10,7 @@ docs_dir: src
nav:
- Python AppImages: index.md
- Developers corner: apps.md
- Developers' corner: apps.md
theme:
name: readthedocs

View File

@@ -10,157 +10,169 @@
{% include "references.md" %}
# Developers corner
# Developers' corner
Python [AppImages][APPIMAGE] are built with the `python-appimage` utility,
available from [PyPI][PYPI]. This utility can also help packaging Python based
applications as AppImages, using an existing Python AppImage and a recipe
Python [AppImages][APPIMAGE] are created using the `python-appimage` utility,
which is available on [PyPI][PYPI]. This utility can also be used to package
Python-based applications as AppImages using an existing AppImage and a recipe
folder.
!!! Caution
The `python-appimage` utility can only package applications that can be
directly installed with `pip`. For more advanced usage, one needs to extract
the Python AppImage and to edit it, e.g. as explained in the [Advanced
installation](index.md#advanced-installation) section. Additional details
on this use case are provided [below](#advanced-packaging).
installed directly with `pip`. For more advanced usage, it is necessary to
extract and edit the Python AppImage, as explained in the [Advanced
installation](index.md#advanced-installation) section. Further details on
this use case can be found [below](#advanced-packaging).
## Building a Python AppImage
The primary scope of `python-appimage` is to relocate an existing Python
installation inside an AppDir, and to build the corresponding AppImage. For
example, the following
The primary purpose of `python-appimage` is to relocate an existing Python
installation to an AppDir and build the corresponding AppImage. For example, the
command
```bash
python-appimage build local -p $(which python2)
```
should build an AppImage of your local Python 2 installation, provided that it
should create an AppImage of your local Python installation, provided that it
exists.
!!! Tip
Help on available arguments and options to `python-appimage` can be obtained
with the `-h` flag. For example, `python-appimage build local -h` provides
help on local builds.
Help on the available arguments and options for `python-appimage` can be
obtained by using the `-h` flag. For example, running
`python-appimage build local -h` provides help on local builds.
{{ begin(".capsule") }}
### Auxiliary tools
The `python-appimage` utility relies on auxiliary tools that are downloaded and
installed at runtime, on need. Those are [appimagetool][APPIMAGETOOL] for
building AppImages, and [patchelf][PATCHELF] in order to edit ELFs runtime paths
(`RPATH`). Auxiliary tools are installed to the the user space. One can get
their location with the `which` command word. For example,
installed on demand during application execution. These are
[appimagetool][APPIMAGETOOL], which is used to build AppImages, and
[patchelf][PATCHELF], which is used to edit runtime paths (`RPATH`) in ELF
files. These auxiliary tools are installed in the application cache. Their
location can be found using the `which` command. For example, the command
```bash
python-appimage which appimagetool
```
returns the location of `appimagetool`, if it has been installed. If not, the
`install` command word can be used in order to trigger its installation.
returns the location of [appimagetool][APPIMAGETOOL] if it has been installed.
If not, the `install` command can be used to trigger its installation.
{{ end(".capsule") }}
## Manylinux Python AppImages
AppImages of your local `python` are unlikely to be portable, except if you run
an ancient Linux distribution. Indeed, a core component preventing portability
across Linuses is the use of different versions of the `glibc` system library.
Hopefully, `glibc` is highly backward compatible. Therefore, a simple
work-around is to compile binaries using the oldest Linux distro you can afford
to. This is the strategy used for creating portable AppImages, as well as for
distributing Python site packages as ready-to-use binary [wheels][WHEELS].
AppImages of your local `python` are unlikely to be portable, unless you are
running an outdated Linux distribution. A core component that prevents
portability across Linux distributions is the use of different versions of the
`glibc` system library. Fortunately, `glibc` is highly backward compatible.
Therefore, a simple workaround is to compile binaries using the oldest Linux
distribution you can. This strategy is used to create portable AppImages and to
distribute Python site packages as ready-to-use binary [wheels][WHEELS].
The Python Packaging Authority (PyPA) has defined standard platform tags for
building Python site packages, labelled [manylinux][MANYLINUX]. These build
platforms are available as Docker images with various versions of Python already
installed. The `python-appimage` utility can be used to package those installs
as AppImages. For example, the following command
building Python site packages labelled [Manylinux][MANYLINUX]. These build
platforms are available as Docker images, with different versions of Python
already installed. The `python-appimage` utility can be used to package these
installations as AppImages. For example, the following command
```bash
python-appimage build manylinux 2014_x86_64 cp310-cp310
python-appimage build manylinux 2014_x86_64 cp313-cp313
```
should build an AppImage of Python 3.10 using the CPython (_cp310-cp310_)
install found in the `manylinux2014_x86_64` Docker image.
should build an AppImage of Python __3.13__ using the CPython (__cp313-cp313__)
installation found in the `manylinux2014_x86_64` Docker image.
!!! Note
Docker needs to be already installed on your system in order to build
Manylinux Python images. However, the command above can be run on the host.
That is, you need **not** to explictly shell inside the manylinux Docker
image.
From version `1.4.0` of `python-appimage` onwards, Docker is **no longer**
required to build the Manylinux Python images. Cross-building is also
supported, for example producing an `aarch64` Python image from an `x86_64`
host.
!!! Warning
Creating multiple Manylinux Python images can significantly increase the
size of the application cache. This can be managed using the
`python-appimage cache` command.
!!! Tip
A compilation of ready-to-use Manylinux Python AppImages is available from
the [releases][RELEASES] area of the `python-appimage` [GitHub
A compilation of ready-to-use Manylinux Python AppImages is available in the
[releases][RELEASES] section of the `python-appimage` [GitHub
repository][GITHUB]. These AppImages are updated weekly, on every Sunday.
!!! Tip
Instead of an AppImage, the `python-appimage build manylinux` command can
produce either an `AppDir` or a bare tarball (i.e. without the AppImage
layer) of a Manylinux Python installation. See the `-b` and `-n` command
line options for more information.
## Simple packaging
The `python-appimage` utility can also be used in order to build simple
applications, that can be `pip` installed. The syntax is
The `python-appimage` utility can also be used to package simple AppImage
applications, whose dependencies can be installed using `pip`. The syntax is
```bash
python-appimage build app -p 3.10 /path/to/recipe/folder
python-appimage build app -p 3.13 /path/to/recipe/folder
```
in order to build a Python 3.10 based application from a recipe folder.
Examples of recipes can be found on GitHub in the [applications][APPLICATIONS]
folder. The recipe folder contains:
to build a Python 3.13-based application from a recipe folder. Examples of
recipes can be found in the [applications][APPLICATIONS] folder on GitHub. The
recipe folder contains
- the AppImage metadata (`application.xml` and `application.desktop`),
- an application icon (e.g. `application.png`),
- a Python requirements file (`requirements.txt`)
- a Python requirements file (`requirements.txt`),
- an entry point script (`entrypoint.sh`).
Additional information on metadata can be found in the AppImage documentation.
That is, for [desktop][APPIMAGE_DESKTOP] and [AppStream XML][APPIMAGE_XML]
files. The `requirements.txt` file allows to specify additional site packages
to be bundled in the AppImage, using `pip`.
Further information on metadata can be found in the AppImage documentation
(e.g., regarding [desktop][APPIMAGE_DESKTOP] and [AppStream XML][APPIMAGE_XML]
files). The `requirements.txt` file enables additional site packages to be
specified for bundling in the AppImage using `pip`.
!!! Caution
For the application to be portable, site packages bundled in the AppImage,
as well as their dependencies, must must be available as binary wheels, or
be pure Python packages.
In order for the application to be portable, the site packages bundled in
the AppImage and their dependencies must be available as binary wheels or
pure Python packages.
If a **C extension** is bundled from **source**, then it will likely **not
be portable**, as further discussed in the [Advanced
If a **C extension** is bundled from **source**, it will likely **not be
portable**; this is discussed further in the [Advanced
packaging](#advanced-packaging) section.
!!! Tip
Some site packages are available only for specific Manylinux tags. This can
be cross-checked by browsing the `Download files` section on the package's
PyPI page.
Some site packages are only available for specific Manylinux tags. You can
check this by browsing the `Download files` section on the package's PyPI
page.
!!! Tip
Since version 1.2, `python-appimage` allows to specify local requirements as
well, using the `local+` tag (see
[PR49](https://github.com/niess/python-appimage/pull/49)). Note however that
this performs a direct copy of the local package, which has several
limitations.
Since version 1.2, `python-appimage` allows local requirements to be
specified using the `local+` tag (see
[PR49](https://github.com/niess/python-appimage/pull/49)). Please note,
however, that this involves directly copying the local package, which has
several limitations.
{{ begin(".capsule") }}
### Entry point script
{% raw %}
The entry point script deserves some additional explanations. This script allows
to customize the startup of your application. A typical `entrypoint.sh` script
would look like
The entry point script deserves some additional explanations. This script lets
you customise your application's startup. A typical `entrypoint.sh` script would
look like this
```bash
{{ python-executable }} ${APPDIR}/opt/python{{ python-version }}/bin/my_app.py "$@"
```
where `my_app.py` is the application startup script, installed by `pip`. As can
be seen from the previous example, the `entrypoint.sh` script recognises some
particular variables, nested between double curly braces, `{{ }}`. Those
variables are listed in the table hereafter. In addition, usual [AppImage
environement variables][APPIMAGE_ENV] can be used as well, if needed. For
example, `$APPDIR` points to the AppImage mount point at runtime.
where `my_app.py` is the application startup script installed by `pip`. As can
be seen from the previous example, the `entrypoint.sh` script recognises
particular variables nested between double curly braces (`{{}}`). These
variables are listed in the table below. In addition, the usual [AppImage
environement variables][APPIMAGE_ENV] can be used if needed. For instance,
`$APPDIR` points to the AppImage mount point at runtime.
{% endraw %}
| variable | Description |
|----------------------|---------------------------------------------------------------|
| `architecture` | The AppImage architecture, e.g. `x86_64`. |
@@ -173,20 +185,20 @@ example, `$APPDIR` points to the AppImage mount point at runtime.
{% raw %}
!!! Note
By default, Python AppImages are not isolated from the user space, nor from
Python specific environment variables, the like `PYTHONPATH`. Depending on
your use case, this can be problematic.
By default, Python AppImages are not isolated from user space or
Python-specific environment variables such as `PYTHONPATH`. Depending on
your use case, this can cause problems.
The runtime isolation level can be changed by adding the `-E`, `-s` or `-I`
options, when invoking the runtime. For example,
`{{ python-executable }} -I` starts a fully isolated Python instance.
You can change the isolation level by adding the `-E`, `-s` or `-I` options
when invoking the runtime. For example, `{{ python-executable }} -I` starts
a fully isolated Python instance.
{% endraw %}
### Bundling data files
`python-appimage` is also capable of bundling in auxilliary data files directly
into the resulting AppImage. `-x/--extra-data` switch exists for that task.
Consider following example.
`python-appimage` is also capable of bundling auxiliary data files directly into
the resulting AppImage. The `-x/--extra-data` switch is used for this purpose.
Consider the following example.
```bash
echo -n "foo" > foo
@@ -195,10 +207,10 @@ echo -n "baz" > bar/baz
python-appimage [your regular parameters] -x foo bar/*
```
User data included in such a way becomes accessible to the Python code
contained within the AppImage in a form of regular files under the directory
pointed to by `APPDIR` environment variable. Example of Python 3 script
that reads these exemplary files is presented below.
In this way, user data becomes accessible to the Python code contained within
the AppImage as regular files under the directory pointed to by the `APPDIR`
environment variable. An example of a Python 3 script that reads these files is
presented below.
```python
import os, pathlib
@@ -206,7 +218,7 @@ for fileName in ("foo", "baz"):
print((pathlib.Path(os.getenv("APPDIR")) / fileName).read_text())
```
Above code, when executed, would print following output.
When executed, the above code would produce the following output.
```bash
foo
@@ -215,37 +227,36 @@ baz
## Advanced packaging
In more complex cases, e.g. if your application relies on external C libraries
not bundled with the Python runtime, then the simple packaging scheme described
previously will fail. Indeed, this falls out of the scope of `python-appimage`,
whose main purpose it to relocate an existing Python install. In this case, you
might rather refer to the initial AppImage [Packaging
Guide][APPIMAGE_PACKAGING], and use alternative tools like
In more complex cases, for example if your application relies on external C
libraries that are not bundled with the Python runtime, the simple packaging
scheme described previously will not work. This falls outside the scope of
`python-appimage`, which is primarily intended for relocating an existing Python
installation. In this case, you may wish to refer to the initial AppImage
[Packaging Guide][APPIMAGE_PACKAGING], and use alternative tools such as
[linuxdeploy][LINUXDEPLOY].
However, `python-appimage` can still be of use in more complex cases by
extracting its AppImages to an AppDir, as discussed in the [Advanced
installation](index.md#advanced-installation) section. The extracted AppImages
contain a relocatable Python runtime, that can be used as a starting base for
building more complex AppImages.
However, `python-appimage` can still be useful in more complex cases, as it can
generate a base AppDir containing a relocatable Python runtime (e.g., using the
`-n` option). This can then serve as a starting point to create more complex
AppImages.
!!! Tip
In some cases, a simple workaround to missing external libraries can be to
fetch portable versions of those from a Manylinux distro, and to bundle them
under `AppDir/usr/lib`. You might also need to edit their dynamic section,
e.g. using [`patchelf`][PATCHELF], which is installed by `python-appimage`.
In some cases, a simple workaround for missing external libraries is to
download portable versions of them from a Manylinux distribution and bundle
them in `AppDir/usr/lib`. You may also need to edit the dynamic section
using [`patchelf`][PATCHELF], which is installed by `python-appimage`.
{{ begin(".capsule") }}
### C extension modules
If your application relies on C extension modules, they need to be compiled on a
Manylinux distro in order to be portable. In addition, their dependencies need
to be bundled as well. In this case, you might better start by building a binary
wheel of your package, using tools like [Auditwheel][AUDITWHEEL] which can
automate some parts of the packaging process. Note that `auditwheel` is already
If your application relies on C extension modules, these must be compiled on a
Manylinux distribution in order to be portable. Their dependencies also need to
be bundled. In this case, it would be better to start by building a binary wheel
of your package using tools like [Auditwheel][AUDITWHEEL], which can automate
some parts of the packaging process. Please note that `auditwheel` is already
installed on the Manylinux Docker images.
Once you have built a binary wheel of your package, it can be used with
`python-appimage` in order to package your application as an AppImage.
Once you have built a binary wheel of your package, you can use it with
`python-appimage` to package your application as an AppImage.
{{ end(".capsule") }}

View File

@@ -15,8 +15,8 @@
# Python AppImages
We provide relocatable Python runtimes for _Linux_ systems, as
[AppImages][APPIMAGE]. These runtimes have been extracted from
[manylinux][MANYLINUX] Docker images.
[AppImages][APPIMAGE]. These runtimes have been extracted from a variety of
[Manylinux][MANYLINUX] Docker images.
{{ "" | id("append-releases-list") }}
## Basic installation
@@ -35,23 +35,24 @@ chmod +x python3.10.2-cp310-cp310-manylinux2014_x86_64.AppImage
!!! Note
As can be seen from the previous [example](#basic-installation-example), the
AppImage name contains several informations. That are, the Python full
version ({{ "3.10.2" | id("example-full-version") }}), the CPython tag
({{ "cp310-cp310" | id("example-python-tag") }}), the Linux compatibility
tag ({{ "manylinux2014" | id("example-linux-tag") }}) and the machine
architecture ({{ "x86_64" | id("example-arch-tag") }}).
AppImage name contains several pieces of information. This includes the
Python full version ({{ "3.10.2" | id("example-full-version") }}), the
[CPython tag][PEP_425] ({{ "cp310-cp310" | id("example-python-tag") }}), the
[Linux compatibility tag][MANYLINUX] ({{ "manylinux2014" |
id("example-linux-tag") }}) and the machine architecture ({{ "x86_64" |
id("example-arch-tag") }}).
!!! Caution
One needs to **select an AppImage** that matches **system requirements**. A
summmary of available Python AppImages is provided at the
[bottom](#available-python-appimages) of this page.
It is essential to **select an AppImage** that aligns with the **system's
specifications**. An overview of the available Python AppImages is provided
at the [bottom](#available-python-appimages) of this page.
{{ begin(".capsule") }}
### Creating a symbolic link
Since AppImages native names are rather lengthy, one might create a symbolic
link, e.g. as
As AppImages' native names are quite lengthy, it might be relevant to create a
symbolic link, for example as
{{ begin("#basic-installation-example-symlink") }}
```bash
@@ -59,23 +60,23 @@ ln -s python3.10.2-cp310-cp310-manylinux2014_x86_64.AppImage python3.10
```
{{ end("#basic-installation-example-symlink") }}
Then, executing the AppImage as
{{ "`./python3.10`" | id("basic-installation-example-execution") }} should
start a Python interactive session on _almost_ any Linux, provided that **fuse**
is supported.
Executing the AppImage as {{ "`./python3.10`" |
id("basic-installation-example-execution") }} should then start a Python
interactive session on almost any Linux distribution, provided that **fuse** is
supported.
{{ end(".capsule") }}
!!! Tip
Fuse is not supported on Windows Subsytem for Linux v1 (WSL1), preventing
AppImages direct execution. Yet, one can still extract the content of Python
AppImages and use them, as explained in the [Advanced
installation](#advanced-installation) section.
Fuse is not supported on Windows Subsystem for Linux v1 (WSL1), which
prevents the direct execution of AppImages. However, it is still possible to
extract the contents of Python AppImages and use them, as explained in the
[Advanced installation](#advanced-installation) section.
## Installing site packages
Site packages can be installed using `pip`, distributed with the AppImage. For
example, the following
Site packages can be installed using `pip`, which is distributed with Python
AppImages. For example, the following command
{{ begin("#site-packages-example") }}
```bash
@@ -83,23 +84,22 @@ example, the following
```
{{ end("#site-packages-example") }}
installs the numpy package, where it is assumed that a symlink to the AppImage
has been previously created. When using the **basic installation** scheme, by
default Python packages are installed to your **user space**, i.e. under
`~/.local` on Linux.
installs the [numpy][NUMPY] package, assuming that a symlink to the AppImage has
been created beforehand. When using this **basic installation** scheme, Python
packages are installed by default to your **user space** (i.e. under `~/.local`
on Linux).
!!! Note
AppImage are read-only. Therefore, site packages cannot be directly
installed to the AppImage. However, the AppImage can be extracted, as
AppImages are read-only. Therefore, site packages cannot be installed
directly to the Python AppImage. However, the AppImage can be extracted, as
explained in the [Advanced installation](#advanced-installation) section.
{{ begin(".capsule") }}
### Alternative site packages location
One can
specify an alternative installation directory for site packages using the
`--target` option of pip. For example, the following
The `--target option` of pip can be used to specify an alternative installation
directory for site packages. For example, the following command
{{ begin("#site-packages-example-target") }}
```bash
@@ -107,7 +107,9 @@ specify an alternative installation directory for site packages using the
```
{{ end("#site-packages-example-target") }}
installs the numpy package besides the AppImage, in a `packages` folder.
installs the [numpy][NUMPY] package in the `packages` folder, besides the
AppImage.
{{ end(".capsule") }}
!!! Tip
@@ -116,33 +118,33 @@ installs the numpy package besides the AppImage, in a `packages` folder.
`PYTHONPATH` environment variable.
!!! Caution
While Python AppImages are relocatable, site packages might not be. In
particular, packages installing executable Python scripts assume a fix
location of the Python runtime. If the Python AppImage is moved, then these
scripts will fail. This can be patched by editing the script
[shebang][SHEBANG], or be reinstalling the corresponding package.
Although Python AppImages are relocatable, site packages may not be. In
particular, packages that install executable Python scripts assume a fixed
location for the Python runtime. If the Python AppImage is moved, these
scripts will fail. This can be resolved by either editing the script
[shebang][SHEBANG] or reinstalling the corresponding package.
## Isolating from the user environment
By default, Python AppImages are not isolated from the user environment. For
example, packages located under `~/.local/lib/pythonX.Y/site-packages` are
loaded prior to AppImage's (system) ones. Note that this is the usual Python
runtime behaviour. However, it can be conflictual for some applications.
loaded before the AppImage's ones. Note that this is the standard Python runtime
behaviour. However, this can be conflictual for some applications.
In order to isolate your application from the user environment, the Python
runtime provides the `-E`, `-s` and `-I` options. For example, invoking a Python
AppImage as {{ "`./python3.10 -s`" | id("user-isolation-example") }} prevents
the loading of user site packages (located under `~/.local`). Additionaly, the
`-E` option disables Python related environment variables. In particular, it
prevents packages under `PYTHONPATH` to be loaded. The `-I` option triggers both
`-E` and `-s`.
To isolate your application from the user environment, the Python runtime
provides the `-E`, `-s` and `-I` options. For example, running {{ "`./python3.10
-s`" | id("user-isolation-example") }} prevents the loading of user site
packages located under `~/.local`. Additionally, the `-E` option disables
Python-related environment variables. In particular, it prevents packages under
`PYTHONPATH` from being loaded. The `-I` option triggers both the `-E` and `-s`
options.
## Using a virtual environement
Isolation can also be achieved with a [virtual environment][VENV]. Python
AppImages can create a `venv` using the standard syntax, e.g. as
[Virtual environments][VENV] can also be used to achieve isolation. For example,
Python AppImages can create a `venv` using the standard syntax, as
{{ begin("#venv-example") }}
```bash
@@ -150,15 +152,16 @@ AppImages can create a `venv` using the standard syntax, e.g. as
```
{{ end("#venv-example") }}
Note that moving the base Python AppImage to another location breaks the virtual
environment. This can be patched by editing symbolic links under `venv/bin`, as
well as the `home` variable in `venv/pyvenv.cfg`. The latter must point to the
AppImage directory.
Please note that moving the base Python AppImage to a different location will
break the virtual environment. This can be resolved by editing the symbolic
links in `venv/bin`, as well as the `home` variable in `venv/pyvenv.cfg`. The
latter must point to the AppImage directory.
!!! Tip
Old Python AppImages, created before version 1.1, fail setting up `pip`
automaticaly during `venv` creation. However, this can be patched by calling
`ensurepip` from within the `venv`, after its creation. For example, as
Old Python AppImages created before version 1.1 fail to set up `pip`
automatically during `venv` creation. However, this can be resolved by
calling `ensurepip` within the virtual environment after its creation. For
example, as
```bash
source /path/to/new/virtual/environment/bin/activate
@@ -170,10 +173,10 @@ python -m ensurepip
## Advanced installation
The [basic installation](#basic-installation) scheme described previously has
some limitations when using Python AppImages as a runtime. For example, site
packages need to be installed to a separate location. This can be solved by
extracting a Python AppImage to an `*.AppDir` directory, e.g. as
certain limitations when Python AppImages are used as the runtime environment.
For example, site packages need to be installed in a different location. This
issue can be resolved by extracting a Python AppImage to an `AppDir`
directory, e.g. as
{{ begin("#advanced-installation-example") }}
```bash
@@ -185,32 +188,32 @@ ln -s python3.10.2-cp310-cp310-manylinux2014_x86_64.AppDir/AppRun python3.10
```
{{ end("#advanced-installation-example") }}
Then, by default **site packages** are installed to the extracted **AppDir**,
when using `pip`. In addition, executable scripts installed by `pip` are patched
in order to use relative [shebangs][SHEBANG]. Consequently, the AppDir can be
freely moved around.
Then, by default, **site packages** are installed to the extracted `AppDir`
when using `pip`. Additionally, executable scripts installed by `pip` are
patched to use relative [shebangs][SHEBANG]. Consequently, the `AppDir` can be
moved around freely.
!!! Note
Python AppDirs follow the [manylinux][MANYLINUX] installation scheme.
Executable scripts are installed under `AppDir/opt/pythonX.Y/bin` where _X_
and _Y_ in _pythonX.Y_ stand for the major and minor version numbers. Site
packages are located under
`AppDir/opt/pythonX.Y/lib/pythonX.Y/site-packages`. For convenience, `pip`
installed applications are also mirrored under `AppDir/usr/bin`, using
symbolic links.
Python `AppDirs` follow the [Manylinux][MANYLINUX] installation scheme.
Executable scripts are installed under the `AppDir/opt/pythonX.Y/bin`
directory, where _X_ and _Y_ represent the major and minor version numbers,
respectively. Site packages are located under
`AppDir/opt/pythonX.Y/lib/pythonX.Y/site-packages`. For convenience,
applications installed using `pip` are also mirrored under `AppDir/usr/bin`
using symbolic links.
!!! Tip
As for Python AppImages, by default the extracted runtime is [not isolated
from the user environment](#isolating-from-the-user-environment). This
As for Python AppImages, the extracted runtime is [not isolated from the
user environment](#isolating-from-the-user-environment) by default. This
behaviour can be changed by editing the `AppDir/usr/bin/pythonX.Y` wrapper
script, and by adding the `-s`, `-E` or `-I` option at the very bottom,
where Python is invoked.
script and adding the `-s`, `-E` or `-I` option to the line invoking Python
(at the end of the script).
{{ begin(".capsule") }}
### Repackaging the AppImage
An extracted AppDir can be re-packaged as an AppImage using
An extracted `AppDir` can be re-packaged as an AppImage using
[appimagetool][APPIMAGETOOL], e.g. as
@@ -227,21 +230,21 @@ chmod +x appimagetool-x86_64.AppImage
```
{{ end("#repackaging-example") }}
This allows to customize your Python AppImage, for example by adding your
preferred site packages.
This allows you to personalise your Python AppImage by adding your preferred
site packages, for example.
{{ end(".capsule") }}
!!! Note
Python AppImages can also be used for packaging Python based applications,
as AppImages. Additional details are provided in the [developers
Python AppImages can also be used to package Python-based applications as
AppImages. Further information can be found in the [developers'
section](apps).
## Available Python AppImages
A summary of available Python AppImages [releases][RELEASES] is provided in the
[table](#appimages-download-links) below. Clicking on a badge should download
the corresponding AppImage.
The [table](#appimages-download-links) below provides a summary of the available
Python AppImage [releases][RELEASES]. Clicking on a badge should download the
corresponding AppImage.
{{ begin("#suggest-appimage-download") }}
!!! Caution

View File

@@ -4,6 +4,7 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
/* Unpack release metadata */
const releases = []
for (const datum of data) {
if (!datum.name.startsWith("Python")) continue;
var full_version = undefined;
const assets = [];
for (const asset of datum.assets) {
@@ -74,7 +75,7 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
elements.push(`<a href="${release.url}">${release.version}</a>`)
}
$("#append-releases-list").html(
" Available Python versions are " +
" The available Python versions are " +
elements.slice(0, -1).join(", ") +
" and " +
elements[elements.length - 1] +
@@ -189,7 +190,11 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
function badge (asset, pad) {
const colors = {i686: "lightgrey", x86_64: "blue"};
const colors = {
aarch64: "d8dee9",
i686: "81a1c1",
x86_64: "5e81ac"
};
const python = asset.python.split("-")[1];
const arch = asset.arch.replace("_", "__");
var color = colors[asset.arch];

59
pyproject.toml Normal file
View File

@@ -0,0 +1,59 @@
[project]
name = "python_appimage"
authors = [
{ name="Valentin Niess", email="valentin.niess@gmail.com" },
]
dynamic = ["version"]
description = "Appimage releases of Python"
readme = "README.md"
requires-python = ">=3.9"
dependencies = [
"requests",
"setuptools",
]
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Programming Language :: Python",
"Topic :: Software Development",
"Operating System :: POSIX :: Linux",
]
license = "GPL-3.0-or-later"
license-files = ["LICENSE"]
[project.scripts]
python-appimage = "python_appimage.__main__:main"
[project.urls]
homepage = "https://github.com/niess/python-appimage"
documentation = "https://python-appimage.readthedocs.io"
download = "https://pypi.python.org/pypi/python-appimage"
source = "https://github.com/niess/python-appimage"
issues = "https://github.com/niess/python-appimage/issues"
[build-system]
requires = ["setuptools >= 77.0.3"]
build-backend = "setuptools.build_meta"
[tool.setuptools.packages.find]
include = ["python_appimage*"]
[tool.setuptools.dynamic]
version = {attr = "python_appimage.__version__"}
[tool.bumpversion]
current_version = "1.4.4"
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
serialize = ["{major}.{minor}.{patch}"]
search = "{current_version}"
replace = "{new_version}"
regex = false
ignore_missing_version = false
tag = false
allow_dirty = false
commit = true
message = "Bump version: v{new_version}"
commit_args = ""
[[tool.bumpversion.files]]
filename = "python_appimage/version.py"

View File

@@ -1,5 +1,7 @@
import platform
from .version import version as __version__
if platform.system() != 'Linux':
raise RuntimeError('invalid system: ' + platform.system())

View File

@@ -1,7 +1,6 @@
import argparse
from importlib import import_module
import os
import sys
__all__ = ['main']
@@ -12,6 +11,7 @@ def exists(path):
raise argparse.ArgumentTypeError("could not find: {}".format(path))
return os.path.abspath(path)
def main():
'''Entry point for the CLI
'''
@@ -34,18 +34,38 @@ def main():
parser.add_argument('-v', '--verbose', help='print extra information',
dest='verbosity', action='store_const', const='DEBUG')
build_parser = subparsers.add_parser('build',
description='Build a Python appimage')
build_subparsers = build_parser.add_subparsers(title='type',
help='Type of AppImage build', dest='sub_command')
cache_parser = subparsers.add_parser('cache',
description='Manage Python appimage cache')
cache_subparsers = cache_parser.add_subparsers(title='operation',
help='Type of cache operation', dest='sub_command')
cache_clean_parser = cache_subparsers.add_parser('clean',
description='Clean cached image(s)')
cache_clean_parser.add_argument('tags', nargs='*',
help='manylinux image tag(s) (e.g. 2014_x86_64)')
cache_clean_parser.add_argument('-a', '--all', action='store_true',
help='remove all image(s) data')
cache_get_parser = cache_subparsers.add_parser('get',
description='Download image(s) to the cache')
cache_get_parser.add_argument('tags', nargs='+',
help='manylinux image tag(s) (e.g. 2014_x86_64)')
cache_get_parser.add_argument('-e', '--extract', action='store_true',
help='extract compressed image data')
cache_list_parser = cache_subparsers.add_parser('list',
description='List cached image(s)')
install_parser = subparsers.add_parser('install',
description='Install binary dependencies')
install_parser.add_argument('binary', nargs='+',
choices=binaries, help='one or more binary name')
build_parser = subparsers.add_parser('build',
description='Build a Python appimage')
build_subparsers = build_parser.add_subparsers(
title='type',
help='Type of AppImage build',
dest='sub_command')
build_local_parser = build_subparsers.add_parser('local',
description='Bundle a local Python installation')
build_local_parser.add_argument('-d', '--destination',
@@ -53,14 +73,18 @@ def main():
build_local_parser.add_argument('-p', '--python', help='python executable')
build_manylinux_parser = build_subparsers.add_parser('manylinux',
description='Bundle a manylinux Python installation using docker')
description='Bundle a manylinux Python installation')
build_manylinux_parser.add_argument('tag',
help='manylinux image tag (e.g. 2010_x86_64)')
build_manylinux_parser.add_argument('abi',
help='python ABI (e.g. cp37-cp37m)')
build_manylinux_parser.add_argument('--contained', help=argparse.SUPPRESS,
action='store_true', default=False)
build_manylinux_parser.add_argument('-b', '--bare',
help='produce a bare image without the AppImage layer',
action='store_true')
build_manylinux_parser.add_argument('-c', '--clean',
help='clean the cache after extraction', action='store_true')
build_manylinux_parser.add_argument('-n', '--no-packaging',
help='do not package (compress) the image', action='store_true')
build_app_parser = build_subparsers.add_parser('app',
description='Build a Python application using a base AppImage')
@@ -72,6 +96,8 @@ def main():
help='linux compatibility tag (e.g. manylinux1_x86_64)')
build_app_parser.add_argument('-n', '--name',
help='application name')
build_app_parser.add_argument('--no-packaging',
help='do not package the app', action='store_true')
build_app_parser.add_argument('--python-tag',
help='python compatibility tag (e.g. cp37-cp37m)')
build_app_parser.add_argument('-p', '--python-version',
@@ -127,5 +153,5 @@ def main():
command.execute(*command._unpack_args(args))
if __name__ == "__main__":
if __name__ == '__main__':
main()

View File

@@ -1,7 +1,7 @@
from .build import build_appimage
from .relocate import cert_file_env_string, patch_binary, relocate_python, \
tcltk_env_string
from .appify import Appifier, tcltk_env_string
from .relocate import patch_binary, relocate_python
__all__ = ['build_appimage', 'cert_file_env_string', 'patch_binary',
'relocate_python', 'tcltk_env_string']
__all__ = ['Appifier', 'build_appimage', 'patch_binary', 'relocate_python',
'tcltk_env_string']

View File

@@ -0,0 +1,270 @@
from dataclasses import dataclass
import glob
import os
from typing import Optional
from ..utils.deps import PREFIX
from ..utils.fs import copy_file, make_tree, remove_file
from ..utils.log import log
from ..utils.template import copy_template, load_template
@dataclass(frozen=True)
class Appifier:
'''Helper class for bundling AppImage specific files'''
'''Path to AppDir root.'''
appdir: str
'''Path to AppDir executables.'''
appdir_bin: str
'''Path to Python executables.'''
python_bin: str
'''Path to Python site-packages.'''
python_pkg: str
'''Tcl/Tk version.'''
tk_version: str
'''Python version.'''
version: 'PythonVersion'
'''Path to SSL certification file.'''
cert_src: Optional[str]=None
def appify(self):
'''Bundle Appimage specific files'''
python_x_y = f'python{self.version.short()}'
pip_x_y = f'pip{self.version.short()}'
# Add a runtime patch for sys.executable, before site.main() execution
log('PATCH', f'{python_x_y} sys.executable')
set_executable_patch(
self.version.short(),
self.python_pkg,
PREFIX + '/data/_initappimage.py'
)
# Set a hook for cleaning sys.path, after site.main() execution
log('HOOK', f'{python_x_y} sys.path')
sitepkgs = self.python_pkg + '/site-packages'
make_tree(sitepkgs)
copy_file(PREFIX + '/data/sitecustomize.py', sitepkgs)
# Symlink SSL certificates
# (see https://github.com/niess/python-appimage/issues/24)
cert_file = '/opt/_internal/certs.pem'
cert_dst = f'{self.appdir}{cert_file}'
if self.cert_src is not None:
if os.path.exists(self.cert_src):
if not os.path.exists(cert_dst):
dirname, basename = os.path.split(cert_dst)
relpath = os.path.relpath(self.cert_src, dirname)
make_tree(dirname)
os.symlink(relpath, cert_dst)
log('INSTALL', basename)
if not os.path.exists(cert_dst):
cert_file = None
# Bundle the python wrapper
wrapper = f'{self.appdir_bin}/{python_x_y}'
if not os.path.exists(wrapper):
log('INSTALL', f'{python_x_y} wrapper')
entrypoint_path = PREFIX + '/data/entrypoint.sh'
entrypoint = load_template(
entrypoint_path,
python=f'python{self.version.flavoured()}'
)
dictionary = {
'entrypoint': entrypoint,
'shebang': '#! /bin/bash',
'tcltk-env': tcltk_env_string(self.python_pkg, self.tk_version)
}
if cert_file:
dictionary['cert-file'] = cert_file_env_string(cert_file)
else:
dictionary['cert-file'] = ''
_copy_template('python-wrapper.sh', wrapper, **dictionary)
# Set or update symlinks to python and pip.
pip_target = f'{self.python_bin}/{pip_x_y}'
if os.path.exists(pip_target):
relpath = os.path.relpath(pip_target, self.appdir_bin)
os.symlink(relpath, f'{self.appdir_bin}/{pip_x_y}')
pythons = glob.glob(self.appdir_bin + '/python?.*')
versions = [os.path.basename(python)[6:] for python in pythons]
latest2, latest3 = '0.0', '0.0'
for version in versions:
if version.startswith('2') and version >= latest2:
latest2 = version
elif version.startswith('3') and version >= latest3:
latest3 = version
if latest2 == self.version.short():
python2 = self.appdir_bin + '/python2'
remove_file(python2)
os.symlink(python_x_y, python2)
has_pip = os.path.exists(self.appdir_bin + '/' + pip_x_y)
if has_pip:
pip2 = self.appdir_bin + '/pip2'
remove_file(pip2)
os.symlink(pip_x_y, pip2)
if latest3 == '0.0':
log('SYMLINK', 'python, python2 to ' + python_x_y)
python = self.appdir_bin + '/python'
remove_file(python)
os.symlink('python2', python)
if has_pip:
log('SYMLINK', 'pip, pip2 to ' + pip_x_y)
pip = self.appdir_bin + '/pip'
remove_file(pip)
os.symlink('pip2', pip)
else:
log('SYMLINK', 'python2 to ' + python_x_y)
if has_pip:
log('SYMLINK', 'pip2 to ' + pip_x_y)
elif latest3 == self.version.short():
log('SYMLINK', 'python, python3 to ' + python_x_y)
python3 = self.appdir_bin + '/python3'
remove_file(python3)
os.symlink(python_x_y, python3)
python = self.appdir_bin + '/python'
remove_file(python)
os.symlink('python3', python)
if os.path.exists(self.appdir_bin + '/' + pip_x_y):
log('SYMLINK', 'pip, pip3 to ' + pip_x_y)
pip3 = self.appdir_bin + '/pip3'
remove_file(pip3)
os.symlink(pip_x_y, pip3)
pip = self.appdir_bin + '/pip'
remove_file(pip)
os.symlink('pip3', pip)
# Bundle the entry point
apprun = f'{self.appdir}/AppRun'
if not os.path.exists(apprun):
log('INSTALL', 'AppRun')
relpath = os.path.relpath(wrapper, self.appdir)
os.symlink(relpath, apprun)
# Bundle the desktop file
desktop_name = f'python{self.version.long()}.desktop'
desktop = os.path.join(self.appdir, desktop_name)
if not os.path.exists(desktop):
log('INSTALL', desktop_name)
apps = 'usr/share/applications'
appfile = f'{self.appdir}/{apps}/{desktop_name}'
if not os.path.exists(appfile):
make_tree(os.path.join(self.appdir, apps))
_copy_template('python.desktop', appfile,
version=self.version.short(),
fullversion=self.version.long())
os.symlink(os.path.join(apps, desktop_name), desktop)
# Bundle icons
icons = 'usr/share/icons/hicolor/256x256/apps'
icon = os.path.join(self.appdir, 'python.png')
if not os.path.exists(icon):
log('INSTALL', 'python.png')
make_tree(os.path.join(self.appdir, icons))
copy_file(PREFIX + '/data/python.png',
os.path.join(self.appdir, icons, 'python.png'))
os.symlink(os.path.join(icons, 'python.png'), icon)
diricon = os.path.join(self.appdir, '.DirIcon')
if not os.path.exists(diricon):
os.symlink('python.png', diricon)
# Bundle metadata
meta_name = f'python{self.version.long()}.appdata.xml'
meta_dir = os.path.join(self.appdir, 'usr/share/metainfo')
meta_file = os.path.join(meta_dir, meta_name)
if not os.path.exists(meta_file):
log('INSTALL', meta_name)
make_tree(meta_dir)
_copy_template(
'python.appdata.xml',
meta_file,
version = self.version.short(),
fullversion = self.version.long()
)
def cert_file_env_string(cert_file):
'''Environment for using a bundled certificate
'''
if cert_file:
return '''
# Export SSL certificate
export SSL_CERT_FILE="${{APPDIR}}{cert_file:}"'''.format(
cert_file=cert_file)
else:
return ''
def _copy_template(name, destination, **kwargs):
path = os.path.join(PREFIX, 'data', name)
copy_template(path, destination, **kwargs)
def tcltk_env_string(python_pkg, tk_version):
'''Environment for using AppImage's TCl/Tk
'''
if tk_version:
return '''
# Export TCl/Tk
export TCL_LIBRARY="${{APPDIR}}/usr/share/tcltk/tcl{tk_version:}"
export TK_LIBRARY="${{APPDIR}}/usr/share/tcltk/tk{tk_version:}"
export TKPATH="${{TK_LIBRARY}}"'''.format(
tk_version=tk_version)
else:
return ''
def set_executable_patch(version, pkgpath, patch):
'''Set a runtime patch for sys.executable name
'''
# This patch needs to be executed before site.main() is called. A natural
# option is to apply it directy to the site module. But, starting with
# Python 3.11, the site module is frozen within Python executable. Then,
# doing so would require to recompile Python. Thus, starting with 3.11 we
# instead apply the patch to the encodings package. Indeed, the latter is
# loaded before the site module, and it is not frozen (as for now).
major, minor = [int(v) for v in version.split('.')]
if (major >= 3) and (minor >= 11):
path = os.path.join(pkgpath, 'encodings', '__init__.py')
else:
path = os.path.join(pkgpath, 'site.py')
with open(path) as f:
source = f.read()
if '_initappimage' in source: return
lines = source.split(os.linesep)
if path.endswith('site.py'):
# Insert the patch before the main function
for i, line in enumerate(lines):
if line.startswith('def main('): break
else:
# Append the patch at end of file
i = len(lines)
with open(patch) as f:
patch = f.read()
lines.insert(i, patch)
lines.insert(i + 1, '')
source = os.linesep.join(lines)
with open(path, 'w') as f:
f.write(source)

View File

@@ -6,25 +6,23 @@ import sys
from ..utils.compat import decode
from ..utils.deps import ensure_appimagetool
from ..utils.docker import docker_run
from ..utils.fs import copy_tree
from ..utils.log import debug, log
from ..utils.tmp import TemporaryDirectory
__all__ = ['build_appimage']
def build_appimage(appdir=None, destination=None):
def build_appimage(appdir=None, *, arch=None, destination=None):
'''Build an AppImage from an AppDir
'''
if appdir is None:
appdir = 'AppDir'
log('BUILD', appdir)
log('BUILD', os.path.basename(appdir))
appimagetool = ensure_appimagetool()
arch = platform.machine()
if arch is None:
arch = platform.machine()
cmd = ['ARCH=' + arch, appimagetool, '--no-appstream', appdir]
if destination is not None:
cmd.append(destination)
@@ -36,7 +34,7 @@ def build_appimage(appdir=None, destination=None):
appimage_pattern = re.compile('should be packaged as ([^ ]+[.]AppImage)')
stdout, appimage = [], None
stdout = []
while True:
out = decode(p.stdout.readline())
stdout.append(out)

View File

@@ -4,71 +4,17 @@ import re
import shutil
import sys
from ..utils.deps import EXCLUDELIST, PATCHELF, PREFIX, ensure_excludelist, \
from .appify import Appifier
from ..manylinux import PythonVersion
from ..utils.deps import EXCLUDELIST, PATCHELF, ensure_excludelist, \
ensure_patchelf
from ..utils.fs import copy_file, copy_tree, make_tree, remove_file, remove_tree
from ..utils.log import debug, log
from ..utils.fs import copy_file, copy_tree, make_tree, remove_file, \
remove_tree
from ..utils.log import log
from ..utils.system import ldd, system
from ..utils.template import copy_template, load_template
__all__ = ["cert_file_env_string", "patch_binary", "relocate_python",
"tcltk_env_string"]
def _copy_template(name, destination, **kwargs):
path = os.path.join(PREFIX, 'data', name)
copy_template(path, destination, **kwargs)
def _get_tk_version(python_pkg):
tkinter = glob.glob(python_pkg + '/lib-dynload/_tkinter*.so')
if tkinter:
tkinter = tkinter[0]
for dep in ldd(tkinter):
name = os.path.basename(dep)
if name.startswith('libtk'):
match = re.search('libtk([0-9]+[.][0-9]+)', name)
return match.group(1)
else:
raise RuntimeError('could not guess Tcl/Tk version')
def _get_tk_libdir(version):
try:
library = system(('tclsh' + version,), stdin='puts [info library]')
except SystemError:
raise RuntimeError('could not locate Tcl/Tk' + version + ' library')
return os.path.dirname(library)
def tcltk_env_string(python_pkg):
'''Environment for using AppImage's TCl/Tk
'''
tk_version = _get_tk_version(python_pkg)
if tk_version:
return '''
# Export TCl/Tk
export TCL_LIBRARY="${{APPDIR}}/usr/share/tcltk/tcl{tk_version:}"
export TK_LIBRARY="${{APPDIR}}/usr/share/tcltk/tk{tk_version:}"
export TKPATH="${{TK_LIBRARY}}"'''.format(
tk_version=tk_version)
else:
return ''
def cert_file_env_string(cert_file):
'''Environment for using a bundled certificate
'''
if cert_file:
return '''
# Export SSL certificate
export SSL_CERT_FILE="${{APPDIR}}{cert_file:}"'''.format(
cert_file=cert_file)
else:
return ''
__all__ = ['patch_binary', 'relocate_python']
_excluded_libs = None
@@ -77,7 +23,7 @@ _excluded_libs = None
def patch_binary(path, libdir, recursive=True):
'''Patch the RPATH of a binary and and fetch its dependencies
'''Patch the RPATH of a binary and fetch its dependencies
'''
global _excluded_libs
@@ -110,54 +56,11 @@ def patch_binary(path, libdir, recursive=True):
continue
target = libdir + '/' + name
if not os.path.exists(target):
libname = os.path.basename(dep)
copy_file(dep, target)
if recursive:
patch_binary(target, libdir, recursive=True)
def set_executable_patch(version, pkgpath, patch):
'''Set a runtime patch for sys.executable name
'''
# This patch needs to be executed before site.main() is called. A natural
# option is to apply it directy to the site module. But, starting with
# Python 3.11, the site module is frozen within Python executable. Then,
# doing so would require to recompile Python. Thus, starting with 3.11 we
# instead apply the patch to the encodings package. Indeed, the latter is
# loaded before the site module, and it is not frozen (as for now).
major, minor = [int(v) for v in version.split('.')]
if (major >= 3) and (minor >= 11):
path = os.path.join(pkgpath, 'encodings', '__init__.py')
else:
path = os.path.join(pkgpath, 'site.py')
with open(path) as f:
source = f.read()
if '_initappimage' in source: return
lines = source.split(os.linesep)
if path.endswith('site.py'):
# Insert the patch before the main function
for i, line in enumerate(lines):
if line.startswith('def main('): break
else:
# Append the patch at end of file
i = len(lines)
with open(patch) as f:
patch = f.read()
lines.insert(i, patch)
lines.insert(i + 1, '')
source = os.linesep.join(lines)
with open(path, 'w') as f:
f.write(source)
def relocate_python(python=None, appdir=None):
'''Bundle a Python install inside an AppDir
'''
@@ -255,9 +158,6 @@ def relocate_python(python=None, appdir=None):
f.write(body)
shutil.copymode(pip_source, target)
relpath = os.path.relpath(target, APPDIR_BIN)
os.symlink(relpath, APPDIR_BIN + '/' + PIP_X_Y)
# Remove unrelevant files
log('PRUNE', '%s packages', PYTHON_X_Y)
@@ -269,17 +169,6 @@ def relocate_python(python=None, appdir=None):
for path in matches:
remove_tree(path)
# Add a runtime patch for sys.executable, before site.main() execution
log('PATCH', '%s sys.executable', PYTHON_X_Y)
set_executable_patch(VERSION, PYTHON_PKG, PREFIX + '/data/_initappimage.py')
# Set a hook for cleaning sys.path, after site.main() execution
log('HOOK', '%s sys.path', PYTHON_X_Y)
sitepkgs = PYTHON_PKG + '/site-packages'
make_tree(sitepkgs)
copy_file(PREFIX + '/data/sitecustomize.py', sitepkgs)
# Set RPATHs and bundle external libraries
log('LINK', '%s C-extensions', PYTHON_X_Y)
@@ -320,111 +209,35 @@ def relocate_python(python=None, appdir=None):
copy_file(cert_file, 'AppDir' + cert_file)
log('INSTALL', basename)
# Bundle AppImage specific files.
appifier = Appifier(
appdir = APPDIR,
appdir_bin = APPDIR_BIN,
python_bin = PYTHON_BIN,
python_pkg = PYTHON_PKG,
tk_version = tk_version,
version = PythonVersion.from_str(FULLVERSION)
)
appifier.appify()
# Bundle the python wrapper
wrapper = APPDIR_BIN + '/' + PYTHON_X_Y
if not os.path.exists(wrapper):
log('INSTALL', '%s wrapper', PYTHON_X_Y)
entrypoint_path = PREFIX + '/data/entrypoint.sh'
entrypoint = load_template(entrypoint_path, python=PYTHON_X_Y)
dictionary = {'entrypoint': entrypoint,
'shebang': '#! /bin/bash',
'tcltk-env': tcltk_env_string(PYTHON_PKG),
'cert-file': cert_file_env_string(cert_file)}
_copy_template('python-wrapper.sh', wrapper, **dictionary)
# Set or update symlinks to python
pythons = glob.glob(APPDIR_BIN + '/python?.*')
versions = [os.path.basename(python)[6:] for python in pythons]
latest2, latest3 = '0.0', '0.0'
for version in versions:
if version.startswith('2') and version >= latest2:
latest2 = version
elif version.startswith('3') and version >= latest3:
latest3 = version
if latest2 == VERSION:
python2 = APPDIR_BIN + '/python2'
remove_file(python2)
os.symlink(PYTHON_X_Y, python2)
has_pip = os.path.exists(APPDIR_BIN + '/' + PIP_X_Y)
if has_pip:
pip2 = APPDIR_BIN + '/pip2'
remove_file(pip2)
os.symlink(PIP_X_Y, pip2)
if latest3 == '0.0':
log('SYMLINK', 'python, python2 to ' + PYTHON_X_Y)
python = APPDIR_BIN + '/python'
remove_file(python)
os.symlink('python2', python)
if has_pip:
log('SYMLINK', 'pip, pip2 to ' + PIP_X_Y)
pip = APPDIR_BIN + '/pip'
remove_file(pip)
os.symlink('pip2', pip)
def _get_tk_version(python_pkg):
tkinter = glob.glob(python_pkg + '/lib-dynload/_tkinter*.so')
if tkinter:
tkinter = tkinter[0]
for dep in ldd(tkinter):
name = os.path.basename(dep)
if name.startswith('libtk'):
match = re.search('libtk([0-9]+[.][0-9]+)', name)
return match.group(1)
else:
log('SYMLINK', 'python2 to ' + PYTHON_X_Y)
if has_pip:
log('SYMLINK', 'pip2 to ' + PIP_X_Y)
elif latest3 == VERSION:
log('SYMLINK', 'python, python3 to ' + PYTHON_X_Y)
python3 = APPDIR_BIN + '/python3'
remove_file(python3)
os.symlink(PYTHON_X_Y, python3)
python = APPDIR_BIN + '/python'
remove_file(python)
os.symlink('python3', python)
if os.path.exists(APPDIR_BIN + '/' + PIP_X_Y):
log('SYMLINK', 'pip, pip3 to ' + PIP_X_Y)
pip3 = APPDIR_BIN + '/pip3'
remove_file(pip3)
os.symlink(PIP_X_Y, pip3)
pip = APPDIR_BIN + '/pip'
remove_file(pip)
os.symlink('pip3', pip)
# Bundle the entry point
apprun = APPDIR + '/AppRun'
if not os.path.exists(apprun):
log('INSTALL', 'AppRun')
relpath = os.path.relpath(wrapper, APPDIR)
os.symlink(relpath, APPDIR + '/AppRun')
# Bundle the desktop file
desktop_name = 'python{:}.desktop'.format(FULLVERSION)
desktop = os.path.join(APPDIR, desktop_name)
if not os.path.exists(desktop):
log('INSTALL', desktop_name)
apps = 'usr/share/applications'
appfile = '{:}/{:}/python{:}.desktop'.format(APPDIR, apps, FULLVERSION)
if not os.path.exists(appfile):
make_tree(os.path.join(APPDIR, apps))
_copy_template('python.desktop', appfile, version=VERSION,
fullversion=FULLVERSION)
os.symlink(os.path.join(apps, desktop_name), desktop)
raise RuntimeError('could not guess Tcl/Tk version')
# Bundle icons
icons = 'usr/share/icons/hicolor/256x256/apps'
icon = os.path.join(APPDIR, 'python.png')
if not os.path.exists(icon):
log('INSTALL', 'python.png')
make_tree(os.path.join(APPDIR, icons))
copy_file(PREFIX + '/data/python.png',
os.path.join(APPDIR, icons, 'python.png'))
os.symlink(os.path.join(icons, 'python.png'), icon)
def _get_tk_libdir(version):
try:
library = system(('tclsh' + version,), stdin='puts [info library]')
except SystemError:
raise RuntimeError('could not locate Tcl/Tk' + version + ' library')
diricon = os.path.join(APPDIR, '.DirIcon')
if not os.path.exists(diricon):
os.symlink('python.png', diricon)
# Bundle metadata
meta_name = 'python{:}.appdata.xml'.format(FULLVERSION)
meta_dir = os.path.join(APPDIR, 'usr/share/metainfo')
meta_file = os.path.join(meta_dir, meta_name)
if not os.path.exists(meta_file):
log('INSTALL', meta_name)
make_tree(meta_dir)
_copy_template('python.appdata.xml', meta_file, version=VERSION,
fullversion=FULLVERSION)
return os.path.dirname(library)

View File

@@ -1,14 +1,14 @@
import json
import glob
import os
from pathlib import Path
import platform
import re
import shutil
import stat
import struct
from ...appimage import build_appimage
from ...utils.compat import decode, find_spec
from ...utils.compat import find_spec
from ...utils.deps import PREFIX
from ...utils.fs import copy_file, copy_tree, make_tree, remove_file, remove_tree
from ...utils.log import log
@@ -27,15 +27,16 @@ def _unpack_args(args):
'''
return args.appdir, args.name, args.python_version, args.linux_tag, \
args.python_tag, args.base_image, args.in_tree_build, \
args.extra_data
args.extra_data, args.no_packaging
_tag_pattern = re.compile('python([^-]+)[-]([^.]+)[.]AppImage')
_linux_pattern = re.compile('manylinux([0-9]+)_' + platform.machine())
def execute(appdir, name=None, python_version=None, linux_tag=None,
python_tag=None, base_image=None, in_tree_build=False,
extra_data=None):
extra_data=None, no_packaging=None):
'''Build a Python application using a base AppImage
'''
@@ -311,8 +312,6 @@ def execute(appdir, name=None, python_version=None, linux_tag=None,
shebang = '#! /bin/bash'
entrypoint = load_template(entrypoint_path, **dictionary)
python_pkg = 'AppDir/opt/python{0:}/lib/python{0:}'.format(
python_version)
dictionary = {'entrypoint': entrypoint,
'shebang': shebang}
if os.path.exists('AppDir/AppRun'):
@@ -322,7 +321,10 @@ def execute(appdir, name=None, python_version=None, linux_tag=None,
# Build the new AppImage
destination = '{:}-{:}.AppImage'.format(application_name,
platform.machine())
build_appimage(destination=destination)
shutil.move(destination, os.path.join(pwd, destination))
fullname = '{:}-{:}'.format(application_name, platform.machine())
if no_packaging:
copy_tree('AppDir', Path(pwd) / fullname)
else:
destination = f'{fullname}.AppImage'
build_appimage(destination=destination)
copy_file(destination, os.path.join(pwd, destination))

View File

@@ -1,13 +1,11 @@
import glob
import os
import platform
import shutil
import sys
from pathlib import Path
import tarfile
from ...appimage import build_appimage, relocate_python
from ...utils.docker import docker_run
from ...utils.fs import copy_tree
from ...utils.manylinux import format_appimage_name, format_tag
from ...appimage import build_appimage
from ...manylinux import ensure_image, PythonExtractor
from ...utils.fs import copy_file, copy_tree
from ...utils.log import log
from ...utils.tmp import TemporaryDirectory
@@ -17,88 +15,55 @@ __all__ = ['execute']
def _unpack_args(args):
'''Unpack command line arguments
'''
return args.tag, args.abi, args.contained
return args.tag, args.abi, args.bare, args.clean, args.no_packaging
def _get_appimage_name(abi, tag):
'''Format the Python AppImage name using the ABI and OS tags
'''
# Read the Python version from the desktop file
desktop = glob.glob('AppDir/python*.desktop')[0]
fullversion = desktop[13:-8]
# Finish building the AppImage on the host. See below.
return format_appimage_name(abi, fullversion, tag)
def execute(tag, abi, contained=False):
'''Build a Python AppImage using a manylinux docker image
def execute(tag, abi, bare=False, clean=False, no_packaging=False):
'''Build a Python AppImage using a Manylinux image
'''
if not contained:
# Forward the build to a Docker image
image = 'quay.io/pypa/' + format_tag(tag)
python = '/opt/python/' + abi + '/bin/python'
image = ensure_image(tag, clean=clean)
pwd = os.getcwd()
dirname = os.path.abspath(os.path.dirname(__file__) + '/../..')
with TemporaryDirectory() as tmpdir:
copy_tree(dirname, 'python_appimage')
pwd = os.getcwd()
with TemporaryDirectory() as tmpdir:
python_extractor = PythonExtractor(
arch = image.arch,
prefix = image.path,
tag = abi
)
appdir = Path(tmpdir) / 'AppDir'
appify = not bare
python_extractor.extract(appdir, appify=appify)
argv = sys.argv[1:]
if argv:
argv = ' '.join(argv)
else:
argv = 'build manylinux {:} {:}'.format(tag, abi)
if tag.startswith("1_"):
# On manylinux1 tk is not installed
script = [
'yum --disablerepo="*" --enablerepo=base install -q -y tk']
else:
# tk is already installed on other platforms
script = []
script += [
python + ' -m python_appimage ' + argv + ' --contained',
''
]
docker_run(image, script)
fullname = '-'.join((
f'{python_extractor.impl}{python_extractor.version.long()}',
abi,
f'{image.tag}_{image.arch}'
))
appimage_name = _get_appimage_name(abi, tag)
if tag.startswith('1_') or tag.startswith('2010_'):
# appimagetool does not run on manylinux1 (CentOS 5) or
# manylinux2010 (CentOS 6). Below is a patch for these specific
# cases.
arch = tag.split('_', 1)[-1]
if arch == platform.machine():
# Pack the image directly from the host
build_appimage(destination=appimage_name)
else:
# Use a manylinux2014 Docker image (CentOS 7) in order to
# pack the image.
script = (
'python -m python_appimage ' + argv + ' --contained',
''
)
docker_run('quay.io/pypa/manylinux2014_' + arch, script)
shutil.move(appimage_name, os.path.join(pwd, appimage_name))
else:
# We are running within a manylinux Docker image
is_manylinux_old = tag.startswith('1_') or tag.startswith('2010_')
if not os.path.exists('AppDir'):
# Relocate the targeted manylinux Python installation
relocate_python()
if no_packaging:
copy_tree(
Path(tmpdir) / 'AppDir',
Path(pwd) / fullname
)
elif bare:
log('COMPRESS', fullname)
destination = f'{fullname}.tar.gz'
tar_path = Path(tmpdir) / destination
with tarfile.open(tar_path, "w:gz") as tar:
tar.add(appdir, arcname=fullname)
copy_file(
tar_path,
Path(pwd) / destination
)
else:
# This is a second stage build. The Docker image has actually been
# overriden (see above).
is_manylinux_old = False
if is_manylinux_old:
# Build only the AppDir when running within a manylinux1 Docker
# image because appimagetool does not support CentOS 5 or CentOS 6.
pass
else:
build_appimage(destination=_get_appimage_name(abi, tag))
destination = f'{fullname}.AppImage'
build_appimage(
appdir = str(appdir),
arch = str(image.arch),
destination = destination
)
copy_file(
Path(tmpdir) / destination,
Path(pwd) / destination
)

View File

64
python_appimage/commands/cache/clean.py vendored Normal file
View File

@@ -0,0 +1,64 @@
import glob
import json
import os
from pathlib import Path
import subprocess
from ...utils.deps import CACHE_DIR
from ...utils.fs import remove_file, remove_tree
from ...utils.log import log
__all__ = ['execute']
def _unpack_args(args):
'''Unpack command line arguments
'''
return (args.tags, args.all)
def execute(images, all_):
'''Clean cached image(s)
'''
cache = Path(CACHE_DIR)
if not images:
images = [image[9:] for image in sorted(os.listdir(cache /
'share/images'))]
for image in images:
try:
image, tag = image.rsplit(':', 1)
except ValueError:
tag = None
if not image.replace('_', '').isalnum():
raise ValueError(f'bad image tag ({image})')
path = cache / f'share/images/manylinux{image}'
if not path.exists():
raise ValueError(f'no such image ({image})')
if tag is None:
if not all_:
path = path / 'extracted'
remove_tree(str(path))
else:
tag_file = path / f'tags/{tag}.json'
if not tag_file.exists():
raise ValueError(f'no such image ({image}:{tag})')
if all_:
with tag_file.open() as f:
layers = json.load(f)["layers"]
for layer in layers:
layer = path / f'layers/{layer}.tar.gz'
if layer.exists():
remove_file(str(layer))
remove_file(str(tag_file))
else:
path = cache / f'share/images/{image}/extracted/{tag}'
if path.exists():
remove_tree(str(path))

18
python_appimage/commands/cache/get.py vendored Normal file
View File

@@ -0,0 +1,18 @@
from ...manylinux import ensure_image
__all__ = ['execute']
def _unpack_args(args):
'''Unpack command line arguments
'''
return (args.tags, args.extract)
def execute(images, extract):
'''Download image(s) to the cache
'''
for image in images:
ensure_image(image, extract=extract)

41
python_appimage/commands/cache/list.py vendored Normal file
View File

@@ -0,0 +1,41 @@
import glob
import os
from pathlib import Path
import subprocess
from ...utils.deps import CACHE_DIR
from ...utils.log import log
__all__ = ['execute']
def _unpack_args(args):
'''Unpack command line arguments
'''
return tuple()
def execute():
'''List cached image(s)
'''
cache = Path(CACHE_DIR)
images = sorted(os.listdir(cache / 'share/images'))
for image in images:
tags = ', '.join((
tag[:-5] for tag in \
sorted(os.listdir(cache / f'share/images/{image}/tags'))
))
if not tags:
continue
path = cache / f'share/images/{image}'
memory = _getsize(path)
log('LIST', f'{image} ({tags}) [{memory}]')
def _getsize(path: Path):
r = subprocess.run(f'du -sh {path}', capture_output=True, check=True,
shell=True)
return r.stdout.decode().split(None, 1)[0]

View File

@@ -1,8 +1,8 @@
import os
import glob
from pathlib import Path
from ..utils.docker import docker_run
from ..manylinux import ensure_image, PythonVersion
from ..utils.log import log
from ..utils.tmp import TemporaryDirectory
__all__ = ['execute']
@@ -18,26 +18,17 @@ def execute(tag):
'''List python versions installed in a manylinux image
'''
with TemporaryDirectory() as tmpdir:
script = (
'for dir in $(ls /opt/python | grep "^cp[0-9]"); do',
' version=$(/opt/python/$dir/bin/python -c "import sys; ' \
'sys.stdout.write(sys.version.split()[0])")',
' echo "$dir $version"',
'done',
)
if tag.startswith('2_'):
image = 'manylinux_' + tag
else:
image = 'manylinux' + tag
result = docker_run(
'quay.io/pypa/' + image,
script,
capture = True
)
pythons = [line.split() for line in result.split(os.linesep) if line]
image = ensure_image(tag)
for (abi, version) in pythons:
log('LIST', "{:7} -> /opt/python/{:}".format(version, abi))
pythons = []
for path in glob.glob(str(image.path / 'opt/python/cp*')):
path = Path(path)
version = PythonVersion.from_str(path.readlink().name[8:]).long()
pythons.append((path.name, version))
pythons = sorted(pythons)
return pythons
n = max(len(version) for (_, version) in pythons)
for (abi, version) in pythons:
log('LIST', "{:{n}} -> /opt/python/{:}".format(version, abi, n=n))
return pythons

View File

@@ -1,7 +1,6 @@
import os
from ..utils import deps
from ..utils.log import log
__all__ = ['execute']

View File

@@ -0,0 +1,52 @@
from types import SimpleNamespace
from .config import Arch, LinuxTag, PythonImpl, PythonVersion
from .download import Downloader
from .extract import ImageExtractor, PythonExtractor
from .patch import Patcher
__all__ = ['Arch', 'Downloader', 'ensure_image', 'ImageExtractor', 'LinuxTag',
'Patcher', 'PythonExtractor', 'PythonImpl', 'PythonVersion']
def ensure_image(tag, *, clean=False, extract=True):
'''Download a manylinux image to the cache'''
try:
tag, image_tag = tag.rsplit(':', 1)
except ValueError:
image_tag = 'latest'
if tag.startswith('2_'):
tag, arch = tag[2:].split('_', 1)
tag = f'2_{tag}'
else:
tag, arch = tag.split('_', 1)
tag = LinuxTag.from_brief(tag)
arch = Arch.from_str(arch)
downloader = Downloader(tag=tag, arch=arch)
downloader.download(tag=image_tag)
if extract:
image_extractor = ImageExtractor(
prefix = downloader.default_destination(),
tag = image_tag
)
image_extractor.extract(clean=clean)
patcher = Patcher(tag=tag, arch=arch)
patcher.patch(destination = image_extractor.default_destination())
return SimpleNamespace(
arch = arch,
tag = tag,
path = image_extractor.default_destination(),
)
else:
return SimpleNamespace(
arch = arch,
tag = tag,
path = downloader.default_destination(),
)

View File

@@ -0,0 +1,109 @@
from enum import auto, Enum
import platform
from typing import NamedTuple, Optional, Union
__all__ = ['Arch', 'PythonImpl', 'PythonVersion']
class Arch(Enum):
'''Supported platform architectures.'''
AARCH64 = auto()
I686 = auto()
X86_64 = auto()
def __str__(self):
return self.name.lower()
@classmethod
def from_host(cls) -> 'Arch':
return cls.from_str(platform.machine())
@classmethod
def from_str(cls, value) -> 'Arch':
for arch in cls:
if value == str(arch):
return arch
else:
raise NotImplementedError(value)
class LinuxTag(Enum):
'''Supported platform tags.'''
MANYLINUX_1 = auto()
MANYLINUX_2010 = auto()
MANYLINUX_2014 = auto()
MANYLINUX_2_24 = auto()
MANYLINUX_2_28 = auto()
def __str__(self):
tag = self.name.lower()
if self in (LinuxTag.MANYLINUX_1, LinuxTag.MANYLINUX_2010,
LinuxTag.MANYLINUX_2014):
return tag.replace('_', '')
else:
return tag
@classmethod
def from_str(cls, value) -> 'LinuxTag':
for tag in cls:
if value == str(tag):
return tag
else:
raise NotImplementedError(value)
@classmethod
def from_brief(cls, value) -> 'LinuxTag':
if value.startswith('2_'):
return cls.from_str('manylinux_' + value)
else:
return cls.from_str('manylinux' + value)
class PythonImpl(Enum):
'''Supported Python implementations.'''
CPYTHON = auto()
def __str__(self):
return 'python'
class PythonVersion(NamedTuple):
''''''
major: int
minor: int
patch: Union[int, str]
flavour: Optional[str]=None
@classmethod
def from_str(cls, value: str) -> 'PythonVersion':
major, minor, patch = value.split('.', 2)
try:
patch, flavour = patch.split('-', 1)
except ValueError:
flavour = None
else:
if flavour == 'nogil':
flavour = 't'
elif flavour == 'ucs2':
flavour = 'm'
elif flavour == 'ucs4':
flavour = 'mu'
else:
raise NotImplementedError(value)
try:
patch = int(patch)
except ValueError:
pass
return cls(int(major), int(minor), patch, flavour)
def flavoured(self) -> str:
flavour = self.flavour if self.flavour == 't' else ''
return f'{self.major}.{self.minor}{flavour}'
def long(self) -> str:
return f'{self.major}.{self.minor}.{self.patch}'
def short(self) -> str:
return f'{self.major}.{self.minor}'

View File

@@ -0,0 +1,167 @@
from dataclasses import dataclass, field
import glob
import hashlib
import json
from pathlib import Path
import requests
import shutil
import tempfile
from typing import Optional
from .config import Arch, LinuxTag
from ..utils.deps import CACHE_DIR
from ..utils.log import debug, log
CHUNK_SIZE = 8189
SUCCESS = 200
class DownloadError(Exception):
pass
@dataclass(frozen=True)
class Downloader:
'''Manylinux tag.'''
tag: LinuxTag
'''Platform architecture.'''
arch: Optional[Arch] = None
'''Docker image.'''
image: str = field(init=False)
'''Authentication token.'''
token: str = field(init=False)
def __post_init__(self):
# Set host arch if not explictly specified.
if self.arch is None:
arch = Arch.from_host()
object.__setattr__(self, 'arch', arch)
# Set image name.
image = f'{self.tag}_{self.arch}'
object.__setattr__(self, 'image', image)
def default_destination(self):
return Path(CACHE_DIR) / f'share/images/{self.image}'
def download(
self,
destination: Optional[Path]=None,
*,
tag: Optional[str] = 'latest'
):
'''Download Manylinux image'''
destination = destination or self.default_destination()
# Authenticate to quay.io.
repository = f'pypa/{self.image}'
log('PULL', f'{self.image}:{tag}')
url = 'https://quay.io/v2/auth'
url = f'{url}?service=quay.io&scope=repository:{repository}:pull'
debug('GET', url)
r = requests.request('GET', url)
if r.status_code == SUCCESS:
object.__setattr__(self, 'token', r.json()['token'])
else:
raise DownloadError(r.status_code, r.text, r.headers)
# Fetch image manifest.
repository = f'pypa/{self.image}'
url = f'https://quay.io/v2/{repository}/manifests/{tag}'
headers = {
'Authorization': f'Bearer {self.token}',
'Accept': 'application/vnd.docker.distribution.manifest.v2+json'
}
debug('GET', url)
r = requests.request('GET', url, headers=headers)
if r.status_code == SUCCESS:
image_digest = r.headers['Docker-Content-Digest'].split(':', 1)[-1]
manifest = r.json()
else:
raise DownloadError(r.status_code, r.text, r.headers)
# Check missing layers to download.
required = [layer['digest'].split(':', 1)[-1] for layer in
manifest['layers']]
missing = []
for hash_ in required:
path = destination / f'layers/{hash_}.tar.gz'
if path.exists():
hasher = hashlib.sha256()
with path.open('rb') as f:
while True:
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
else:
hasher.update(chunk)
h = hasher.hexdigest()
if h != hash_:
missing.append(hash_)
else:
debug('FOUND', f'{hash_}.tar.gz')
else:
missing.append(hash_)
# Fetch missing layers.
with tempfile.TemporaryDirectory() as tmpdir:
workdir = Path(tmpdir)
for i, hash_ in enumerate(missing):
debug('DOWNLOAD', f'{self.image}:{tag} '
f'[{i + 1} / {len(missing)}]')
filename = f'{hash_}.tar.gz'
url = f'https://quay.io/v2/{repository}/blobs/sha256:{hash_}'
debug('GET', url)
r = requests.request('GET', url, headers=headers, stream=True)
if r.status_code == SUCCESS:
debug('STREAM', filename)
else:
raise DownloadError(r.status_code, r.text, r.headers)
hasher = hashlib.sha256()
tmp = workdir / 'layer.tgz'
with open(tmp, "wb") as f:
for chunk in r.iter_content(CHUNK_SIZE):
if chunk:
f.write(chunk)
hasher.update(chunk)
h = hasher.hexdigest()
if h != hash_:
raise DownloadError(
f'bad hash (expected {hash_}, found {h})'
)
layers_dir = destination / 'layers'
layers_dir.mkdir(exist_ok=True, parents=True)
shutil.move(tmp, layers_dir / filename)
tags_dir = destination / 'tags'
tags_dir.mkdir(exist_ok=True, parents=True)
with open(tags_dir / f'{tag}.json', "w") as f:
json.dump({'digest': image_digest, 'layers': required}, f)
# Remove unused layers.
required = set(required)
for tag in glob.glob(str(destination / 'tags/*.json')):
with open(tag) as f:
tag = json.load(f)
required |= set(tag["layers"])
required = [f'{hash_}.tar.gz' for hash_ in required]
for layer in glob.glob(str(destination / 'layers/*.tar.gz')):
layer = Path(layer)
if layer.name not in required:
debug('REMOVE', f'{self.image} [layer/{layer.stem}]')
layer.unlink()

View File

@@ -0,0 +1,418 @@
import atexit
from dataclasses import dataclass, field
from distutils.version import LooseVersion
import glob
import json
import os
import re
from pathlib import Path
import shutil
import stat
import subprocess
from typing import Dict, List, Optional
from .config import Arch, PythonImpl, PythonVersion
from ..appimage import Appifier
from ..utils.deps import ensure_excludelist, ensure_patchelf, EXCLUDELIST, \
PATCHELF
from ..utils.log import debug, log
@dataclass(frozen=True)
class PythonExtractor:
'''Python extractor from an extracted Manylinux image.'''
arch: Arch
'''Target architecture'''
prefix: Path
'''Target image path'''
tag: str
'''Python binary tag'''
excludelist: Optional[Path] = None
'''Exclude list for shared libraries.'''
patchelf: Optional[Path] = None
'''Patchelf executable.'''
excluded: List[str] = field(init=False)
'''Excluded shared libraries.'''
impl: PythonImpl = field(init=False)
'''Python implementation'''
library_path: List[str] = field(init=False)
'''Search paths for libraries (LD_LIBRARY_PATH)'''
python_prefix: Path = field(init=False)
'''Python installation prefix'''
version: PythonVersion = field(init=False)
'''Python version'''
def __post_init__(self):
# Locate Python installation.
link = os.readlink(self.prefix / f'opt/python/{self.tag}')
if not link.startswith('/'):
raise NotImplementedError()
object.__setattr__(self, 'python_prefix', self.prefix / link[1:])
# Parse implementation and version.
head, tail = Path(link).name.split('-', 1)
if head == 'cpython':
impl = PythonImpl.CPYTHON
version = PythonVersion.from_str(tail)
else:
raise NotImplementedError()
object.__setattr__(self, 'impl', impl)
object.__setattr__(self, 'version', version)
# Set libraries search path.
paths = []
if self.arch in (Arch.AARCH64, Arch.X86_64):
paths.append(self.prefix / 'lib64')
paths.append(self.prefix / 'usr/lib64')
if self.arch == Arch.X86_64:
paths.append(self.prefix / 'lib/x86_64-linux-gnu')
paths.append(self.prefix / 'usr/lib/x86_64-linux-gnu')
else:
paths.append(self.prefix / 'lib/aarch64-linux-gnu')
paths.append(self.prefix / 'usr/lib/aarch64-linux-gnu')
elif self.arch == Arch.I686:
paths.append(self.prefix / 'lib')
paths.append(self.prefix / 'usr/lib')
paths.append(self.prefix / 'lib/i386-linux-gnu')
paths.append(self.prefix / 'usr/lib/i386-linux-gnu')
else:
raise NotImplementedError()
paths.append(self.prefix / 'usr/local/lib')
patterns = (
'curl-*',
'mpdecimal-*',
'openssl-*',
'sqlite*',
)
for pattern in patterns:
pattern = str(self.prefix / f'opt/_internal/{pattern}/lib')
for match in glob.glob(pattern):
paths.append(Path(match))
object.__setattr__(self, 'library_path', paths)
# Set excluded libraries.
if self.excludelist:
excludelist = Path(self.excludelist)
else:
ensure_excludelist()
excludelist = Path(EXCLUDELIST)
excluded = set()
with excludelist.open() as f:
for line in f:
line = line.strip()
if line and not line.startswith('#'):
excluded.add(line)
excluded.add('ld-linux-aarch64.so.1') # patch for aarch64.
object.__setattr__(self, 'excluded', excluded)
# Set patchelf, if not provided.
if self.patchelf is None:
ensure_patchelf()
object.__setattr__(self, 'patchelf', PATCHELF)
else:
assert(self.patchelf.exists())
def extract(
self,
destination: Path,
*,
appify: Optional[bool]=False,
python_prefix: Optional[str]=None,
system_prefix: Optional[str]=None,
):
'''Extract Python runtime.'''
python = f'python{self.version.short()}'
flavoured_python = f'python{self.version.flavoured()}'
runtime = f'bin/{flavoured_python}'
packages = f'lib/{flavoured_python}'
pip = f'bin/pip{self.version.short()}'
if python_prefix is None:
python_prefix = f'opt/{flavoured_python}'
if system_prefix is None:
system_prefix = 'usr'
python_dest = destination / python_prefix
system_dest = destination / system_prefix
# Locate include files.
include = glob.glob(str(self.python_prefix / 'include/*'))
if include:
include = Path(include[0]).name
include = f'include/{include}'
else:
raise NotImplementedError()
# Clone Python runtime.
log('CLONE',
f'{python} from {self.python_prefix.relative_to(self.prefix)}')
(python_dest / 'bin').mkdir(exist_ok=True, parents=True)
shutil.copy(self.python_prefix / runtime, python_dest / runtime)
# Clone pip wrapper.
with open(self.python_prefix / pip) as f:
f.readline() # Skip shebang.
body = f.read()
with open(python_dest / pip, 'w') as f:
f.write('#! /bin/sh\n')
f.write(' '.join((
'"exec"',
f'"$(dirname $(readlink -f ${0}))/{flavoured_python}"',
'"$0"',
'"$@"\n'
)))
f.write(body)
shutil.copymode(self.python_prefix / pip, python_dest / pip)
# Clone Python packages.
for folder in (packages, include):
shutil.copytree(self.python_prefix / folder, python_dest / folder,
symlinks=True, dirs_exist_ok=True)
# Remove some clutters.
log('PRUNE', '%s packages', python)
shutil.rmtree(python_dest / packages / 'test', ignore_errors=True)
for root, dirs, files in os.walk(python_dest / packages):
root = Path(root)
for d in dirs:
if d == '__pycache__':
shutil.rmtree(root / d, ignore_errors=True)
for f in files:
if f.endswith('.pyc'):
(root / f).unlink()
# Map binary dependencies.
libs = self.ldd(self.python_prefix / f'bin/{flavoured_python}')
path = Path(self.python_prefix / f'{packages}/lib-dynload')
for module in glob.glob(str(path / "*.so")):
l = self.ldd(module)
libs.update(l)
# Copy and patch binary dependencies.
libdir = system_dest / 'lib'
libdir.mkdir(exist_ok=True, parents=True)
for (name, src) in libs.items():
dst = libdir / name
shutil.copy(src, dst, follow_symlinks=True)
# Some libraries are read-only, which prevents overriding the
# destination directory. Below, we change the permission of
# destination files to read-write (for the owner).
mode = dst.stat().st_mode
if not (mode & stat.S_IWUSR):
mode = mode | stat.S_IWUSR
dst.chmod(mode)
self.set_rpath(dst, '$ORIGIN')
# Patch RPATHs of binary modules.
log('LINK', '%s C-extensions', python)
path = Path(python_dest / f'{packages}/lib-dynload')
for module in glob.glob(str(path / "*.so")):
src = Path(module)
dst = os.path.relpath(libdir, src.parent)
self.set_rpath(src, f'$ORIGIN/{dst}')
# Patch RPATHs of Python runtime.
src = python_dest / runtime
dst = os.path.relpath(libdir, src.parent)
self.set_rpath(src, f'$ORIGIN/{dst}')
# Copy SSL certificates (i.e. clone certifi).
certs = self.prefix / 'opt/_internal/certs.pem'
if certs.is_symlink():
dst = self.prefix / str(certs.readlink())[1:]
certifi = dst.parent
assert(certifi.name == 'certifi')
site_packages = certifi.parent
assert(site_packages.name == 'site-packages')
log('INSTALL', certifi.name)
matches = [
Path(src) for src in glob.glob(str(site_packages / 'certifi*'))
]
matches = sorted(matches, key=lambda src: src.name)
cert_src = None
for src in matches:
dst = python_dest / f'{packages}/site-packages/{src.name}'
if not dst.exists():
shutil.copytree(src, dst, symlinks=True)
if cert_src is None:
cacert_pem = dst / 'cacert.pem'
if cacert_pem.exists():
cert_src = cacert_pem
assert(cert_src is not None)
else:
raise NotImplementedError()
# Copy Tcl & Tk data.
tx_version = []
for match in glob.glob(str(system_dest / 'lib/libtk*')):
path = system_dest / f'lib/{match}'
tx_version.append(LooseVersion(path.name[5:8]))
if tx_version:
tx_version.sort()
tx_version = tx_version[-1]
for location in ('usr/local/lib', 'usr/share', 'usr/share/tcltk'):
tcltk_src = self.prefix / location
path = tcltk_src / f'tk{tx_version}'
if path.exists() and path.is_dir():
break
else:
raise ValueError(f'could not locate Tcl/Tk{tx_version}')
log('INSTALL', f'Tcl/Tk{tx_version}')
tcltk_dir = Path(system_dest / 'share/tcltk')
tcltk_dir.mkdir(exist_ok=True, parents=True)
for tx in ('tcl', 'tk'):
name = f'{tx}{tx_version}'
src = tcltk_src / name
dst = tcltk_dir / name
shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True)
if appify:
appifier = Appifier(
appdir = str(destination),
appdir_bin = str(system_dest / 'bin'),
python_bin = str(python_dest / 'bin'),
python_pkg = str(python_dest / packages),
version = self.version,
tk_version = tx_version,
cert_src = cert_src
)
appifier.appify()
def ldd(self, target: Path) -> Dict[str, Path]:
'''Cross-platform implementation of ldd, using readelf.'''
pattern = re.compile(r'[(]NEEDED[)]\s+Shared library:\s+\[([^\]]+)\]')
dependencies = dict()
def recurse(target: Path):
result = subprocess.run(f'readelf -d {target}', shell=True,
check=True, capture_output=True)
stdout = result.stdout.decode()
matches = pattern.findall(stdout)
for match in matches:
if (match not in dependencies) and (match not in self.excluded):
path = self.locate_library(match)
dependencies[match] = path
recurse(path)
recurse(target)
return dependencies
def locate_library(self, name: str) -> Path:
'''Locate a library given its qualified name.'''
for dirname in self.library_path:
path = dirname / name
if path.exists():
return path
else:
raise FileNotFoundError(name)
def set_rpath(self, target, rpath):
cmd = f'{self.patchelf} --print-rpath {target}'
result = subprocess.run(cmd, shell=True, check=True,
capture_output=True)
current_rpath = result.stdout.decode().strip()
if current_rpath != rpath:
cmd = f"{self.patchelf} --set-rpath '{rpath}' {target}"
subprocess.run(cmd, shell=True, check=True, capture_output=True)
@dataclass(frozen=True)
class ImageExtractor:
'''Manylinux image extractor from layers.'''
prefix: Path
'''Manylinux image prefix.'''
tag: Optional[str] = 'latest'
'''Manylinux image tag.'''
def default_destination(self):
return self.prefix / f'extracted/{self.tag}'
def extract(self, destination: Optional[Path]=None, *, clean=False):
'''Extract Manylinux image.'''
if destination is None:
destination = self.default_destination()
if clean:
def clean(destination):
shutil.rmtree(destination, ignore_errors=True)
atexit.register(clean, destination)
log('EXTRACT', f'{self.prefix.name}:{self.tag}')
with open(self.prefix / f'tags/{self.tag}.json') as f:
meta = json.load(f)
layers = meta['layers']
extracted = []
extracted_file = destination / '.extracted'
if destination.exists():
clean_destination = True
if extracted_file.exists():
with extracted_file.open() as f:
extracted = f.read().split(os.linesep)[:-1]
for a, b in zip(layers, extracted):
if a != b:
break
else:
clean_destination = False
if clean_destination:
shutil.rmtree(destination, ignore_errors=True)
for i, layer in enumerate(layers):
try:
if layer == extracted[i]:
continue
except IndexError:
pass
debug('EXTRACT', f'{layer}.tar.gz')
filename = self.prefix / f'layers/{layer}.tar.gz'
cmd = ''.join((
f'trap \'chmod u+rw -R {destination}\' EXIT ; ',
f'mkdir -p {destination} && ',
f'tar -xzf {filename} --exclude=dev -C {destination} && ',
f'echo \'{layer}\' >> {extracted_file}'
))
r = subprocess.run(f'/bin/bash -c "{cmd}"', shell=True,
capture_output=True)
if r.returncode != 0:
raise ValueError(r.stderr.decode())

View File

@@ -0,0 +1,48 @@
from dataclasses import dataclass
from pathlib import Path
import os
import stat
import subprocess
from typing import Optional
from .config import Arch, LinuxTag
from ..utils.deps import CACHE_DIR
from ..utils.log import debug, log
from ..utils.url import urlretrieve
@dataclass(frozen=True)
class Patcher:
'''Manylinux tag.'''
tag: LinuxTag
'''Platform architecture.'''
arch: Optional[Arch] = None
def patch(self, destination: Path):
'''Apply any patch'''
cache = Path(CACHE_DIR) / f'share/patches/'
if self.tag == LinuxTag.MANYLINUX_1:
patch = f'tk-manylinux1_{self.arch}'
log('PATCH', patch)
tarfile = f'{patch}.tar.gz'
path = cache / tarfile
if not path.exists():
url = f'https://github.com/niess/python-appimage/releases/download/manylinux1/{tarfile}'
urlretrieve(url, path)
mode = os.stat(path)[stat.ST_MODE]
os.chmod(path, mode | stat.S_IWGRP | stat.S_IWOTH)
debug('EXTRACT', tarfile)
cmd = ''.join((
f'trap \'chmod u+rw -R {destination}\' EXIT ; ',
f'mkdir -p {destination} && ',
f'tar -xzf {path} -C {destination}',
))
r = subprocess.run(f'/bin/bash -c "{cmd}"', shell=True,
capture_output=True)
if r.returncode != 0:
raise ValueError(r.stderr.decode())

View File

@@ -9,28 +9,30 @@ from .tmp import TemporaryDirectory
from .url import urlretrieve
__all__ = ['APPIMAGETOOL', 'EXCLUDELIST', 'PATCHELF', 'PREFIX',
'ensure_appimagetool', 'ensure_excludelist', 'ensure_patchelf']
_ARCH = platform.machine()
CACHE_DIR = os.path.expanduser('~/.cache/python-appimage')
'''Package cache location'''
PREFIX = os.path.abspath(os.path.dirname(__file__) + '/..')
'''Package installation prefix'''
APPIMAGETOOL_DIR = os.path.expanduser('~/.local/bin')
APPIMAGETOOL_DIR = os.path.join(CACHE_DIR, 'bin')
'''Location of the appimagetool binary'''
APPIMAGETOOL_VERSION = '12'
APPIMAGETOOL_VERSION = 'continuous'
'''Version of the appimagetool binary'''
EXCLUDELIST = PREFIX + '/data/excludelist'
EXCLUDELIST = os.path.join(CACHE_DIR, 'share/excludelist')
'''AppImage exclusion list'''
PATCHELF = os.path.expanduser('~/.local/bin/patchelf')
PATCHELF = os.path.join(CACHE_DIR, 'bin/patchelf')
'''Location of the PatchELF binary'''
PATCHELF_VERSION = '0.14.3'
'''Version of the patchelf binary'''
def ensure_appimagetool(dry=False):
'''Fetch appimagetool from the web if not available locally
'''
@@ -39,7 +41,6 @@ def ensure_appimagetool(dry=False):
appimagetool_name = 'appimagetool'
else:
appimagetool_name = 'appimagetool-' + APPIMAGETOOL_VERSION
appimagetool = os.path.join(APPIMAGETOOL_DIR, appimagetool_name)
appdir_name = '.'.join(('', appimagetool_name, 'appdir', _ARCH))
appdir = os.path.join(APPIMAGETOOL_DIR, appdir_name)
apprun = os.path.join(appdir, 'AppRun')
@@ -91,19 +92,18 @@ def ensure_patchelf():
if os.path.exists(PATCHELF):
return False
iarch = 'i386' if _ARCH == 'i686' else _ARCH
appimage = 'patchelf-{0:}.AppImage'.format(iarch)
baseurl = 'https://github.com/niess/patchelf.appimage/releases/download'
tgz = '-'.join(('patchelf', PATCHELF_VERSION, _ARCH)) + '.tar.gz'
baseurl = 'https://github.com/NixOS/patchelf'
log('INSTALL', 'patchelf from %s', baseurl)
dirname = os.path.dirname(PATCHELF)
patchelf = dirname + '/patchelf'
make_tree(dirname)
with TemporaryDirectory() as tmpdir:
urlretrieve(os.path.join(baseurl, 'rolling', appimage), appimage)
os.chmod(appimage, stat.S_IRWXU)
system(('./' + appimage, '--appimage-extract'))
copy_file('squashfs-root/usr/bin/patchelf', patchelf)
urlretrieve(os.path.join(baseurl, 'releases', 'download',
PATCHELF_VERSION, tgz), tgz)
system(('tar', 'xzf', tgz))
copy_file('bin/patchelf', patchelf)
os.chmod(patchelf, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
return True

View File

@@ -1,58 +0,0 @@
import os
import platform
import stat
import subprocess
import sys
from .compat import decode
from .log import log
from .system import system
def docker_run(image, extra_cmds, capture=False):
'''Execute commands within a docker container
'''
ARCH = platform.machine()
if image.endswith(ARCH):
bash_arg = '/pwd/run.sh'
elif image.endswith('i686') and ARCH == 'x86_64':
bash_arg = '-c "linux32 /pwd/run.sh"'
elif image.endswith('x86_64') and ARCH == 'i686':
bash_arg = '-c "linux64 /pwd/run.sh"'
else:
raise ValueError('Unsupported Docker image: ' + image)
log('PULL', image)
system(('docker', 'pull', image))
script = [
'set -e',
'trap "chown -R {:}:{:} *" EXIT'.format(os.getuid(),
os.getgid()),
'cd /pwd'
]
script += extra_cmds
with open('run.sh', 'w') as f:
f.write(os.linesep.join(script))
os.chmod('run.sh', stat.S_IRWXU)
cmd = ' '.join(('docker', 'run', '--mount',
'type=bind,source={:},target=/pwd'.format(os.getcwd()),
image, '/bin/bash', bash_arg))
if capture:
opts = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
else:
opts = {}
log('RUN', image)
p = subprocess.Popen(cmd, shell=True, **opts)
r = p.communicate()
if p.returncode != 0:
if p.returncode == 139:
sys.stderr.write("segmentation fault when running Docker (139)\n")
sys.exit(p.returncode)
if capture:
return decode(r[0])

View File

@@ -28,10 +28,14 @@ def urlretrieve(url, filename=None):
'''
if filename is None:
filename = os.path.basename(url)
debug('DOWNLOAD', '%s from %s', name, os.path.dirname(url))
debug('DOWNLOAD', '%s from %s', filename, os.path.dirname(url))
else:
debug('DOWNLOAD', '%s as %s', url, filename)
parent_directory = os.path.dirname(filename)
if parent_directory and not os.path.exists(parent_directory):
os.makedirs(parent_directory)
if _urlretrieve is None:
data = urllib2.urlopen(url).read()
with open(filename, 'w') as f:

View File

@@ -0,0 +1 @@
version = '1.4.4'

319
scripts/test-appimage.py Executable file
View File

@@ -0,0 +1,319 @@
#! /usr/bin/env python3
import argparse
from enum import auto, Enum
import inspect
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from types import FunctionType
from typing import NamedTuple
from python_appimage.manylinux import PythonVersion
ARGS = None
def assert_eq(expected, found):
if expected != found:
raise AssertionError('expected "{}", found "{}"'.format(
expected, found))
class Script(NamedTuple):
'''Python script wrapper'''
content: str
def run(self, appimage: Path):
'''Run the script through an appimage'''
with tempfile.TemporaryDirectory() as tmpdir:
script = f'{tmpdir}/script.py'
with open(script, 'w') as f:
f.write(inspect.getsource(assert_eq))
f.write(os.linesep)
f.write(self.content)
return system(f'{appimage} {script}')
class Status(Enum):
'''Test exit status'''
FAILED = auto()
SKIPPED = auto()
SUCCESS = auto()
def __str__(self):
return self.name
def system(cmd):
'''Run a system command'''
r = subprocess.run(cmd, capture_output=True, shell=True)
if r.returncode != 0:
raise ValueError(r.stderr.decode())
else:
return r.stdout.decode()
class TestContext:
'''Context for testing an image'''
def __init__(self, appimage):
self.appimage = appimage
# Guess python version from appimage name.
version, _, abi, *_ = appimage.name.split('-', 3)
version = version[6:]
if abi.endswith('t'):
version += '-nogil'
self.version = PythonVersion.from_str(version)
# Get some specific AppImage env variables.
self.env = eval(Script('''
import os
appdir = os.environ['APPDIR']
env = {}
for var in ('SSL_CERT_FILE', 'TCL_LIBRARY', 'TK_LIBRARY', 'TKPATH'):
try:
env[var] = os.environ[var].replace(appdir, '$APPDIR')
except KeyError:
pass
print(env)
''').run(appimage))
# Extract the AppImage.
tmpdir = tempfile.TemporaryDirectory()
dst = Path(tmpdir.name) / appimage.name
shutil.copy(appimage, dst)
system(f'cd {tmpdir.name} && ./{appimage.name} --appimage-extract')
self.appdir = Path(tmpdir.name) / 'squashfs-root'
self.tmpdir = tmpdir
def list_content(self, path=None):
'''List the content of an extracted directory'''
path = self.appdir if path is None else self.appdir / path
return sorted(os.listdir(path))
def run(self):
'''Run all tests'''
tests = []
for key, value in self.__class__.__dict__.items():
if isinstance(value, FunctionType):
if key.startswith('test_'):
tests.append(value)
n = len(tests)
m = max(len(test.__doc__) for test in tests)
for i, test in enumerate(tests):
sys.stdout.write(
f'[ {self.appimage.name} | {i + 1:2}/{n} ] {test.__doc__:{m}}'
)
sys.stdout.flush()
try:
status = test(self)
except Exception as e:
status = Status.FAILED
sys.stdout.write(
f' -> {status} ({test.__name__}){os.linesep}')
sys.stdout.flush()
raise e
else:
sys.stdout.write(f' -> {status}{os.linesep}')
sys.stdout.flush()
def test_root_content(self):
'''Check the appimage root content'''
content = self.list_content()
expected = ['.DirIcon', 'AppRun', 'opt', 'python.png',
f'python{self.version.long()}.desktop', 'usr']
assert_eq(expected, content)
return Status.SUCCESS
def test_python_content(self):
'''Check the appimage python content'''
prefix = f'opt/python{self.version.flavoured()}'
content = self.list_content(prefix)
assert_eq(['bin', 'include', 'lib'], content)
content = self.list_content(f'{prefix}/bin')
assert_eq(
[f'pip{self.version.short()}', f'python{self.version.flavoured()}'],
content
)
content = self.list_content(f'{prefix}/include')
if (self.version.major == 3) and (self.version.minor <= 7):
expected = [f'python{self.version.short()}m']
else:
expected = [f'python{self.version.flavoured()}']
assert_eq(expected, content)
content = self.list_content(f'{prefix}/lib')
assert_eq([f'python{self.version.flavoured()}'], content)
return Status.SUCCESS
def test_system_content(self):
'''Check the appimage system content'''
content = self.list_content('usr')
assert_eq(['bin', 'lib', 'share'], content)
content = self.list_content('usr/bin')
expected = [
'pip', f'pip{self.version.major}', f'pip{self.version.short()}',
'python', f'python{self.version.major}',
f'python{self.version.short()}'
]
assert_eq(expected, content)
return Status.SUCCESS
def test_tcltk_bundling(self):
'''Check Tcl/Tk bundling'''
if 'TK_LIBRARY' not in self.env:
return Status.SKIPPED
else:
for var in ('TCL_LIBRARY', 'TK_LIBRARY', 'TKPATH'):
path = Path(self.env[var].replace('$APPDIR', str(self.appdir)))
assert path.exists()
return Status.SUCCESS
def test_ssl_bundling(self):
'''Check SSL certs bundling'''
var = 'SSL_CERT_FILE'
path = Path(self.env[var].replace('$APPDIR', str(self.appdir)))
assert path.exists()
return Status.SUCCESS
def test_bin_symlinks(self):
'''Check /usr/bin symlinks'''
assert_eq(
(self.appdir /
f'opt/python{self.version.flavoured()}/bin/pip{self.version.short()}'),
(self.appdir / f'usr/bin/pip{self.version.short()}').resolve()
)
assert_eq(
f'pip{self.version.short()}',
str((self.appdir / f'usr/bin/pip{self.version.major}').readlink())
)
assert_eq(
f'pip{self.version.major}',
str((self.appdir / 'usr/bin/pip').readlink())
)
assert_eq(
f'python{self.version.short()}',
str((self.appdir / f'usr/bin/python{self.version.major}').readlink())
)
assert_eq(
f'python{self.version.major}',
str((self.appdir / 'usr/bin/python').readlink())
)
return Status.SUCCESS
def test_appimage_hook(self):
'''Test the appimage hook'''
Script(f'''
import os
assert_eq(os.environ['APPIMAGE_COMMAND'], '{self.appimage}')
import sys
assert_eq('{self.appimage}', sys.executable)
assert_eq('{self.appimage}', sys._base_executable)
''').run(self.appimage)
return Status.SUCCESS
def test_python_prefix(self):
'''Test the python prefix'''
Script(f'''
import os
import sys
expected = os.environ["APPDIR"] + '/opt/python{self.version.flavoured()}'
assert_eq(expected, sys.prefix)
''').run(self.appimage)
return Status.SUCCESS
def test_ssl_request(self):
'''Test SSL request (see issue #24)'''
if self.version.major == 2:
return Status.SKIPPED
else:
Script('''
from http import HTTPStatus
import urllib.request
with urllib.request.urlopen('https://wikipedia.org') as r:
assert_eq(r.status, HTTPStatus.OK)
''').run(self.appimage)
return Status.SUCCESS
def test_pip_install(self):
'''Test pip installing to an extracted AppImage'''
r = system(f'{self.appdir}/AppRun -m pip install pip-install-test')
assert('Successfully installed pip-install-test' in r)
path = self.appdir / f'opt/python{self.version.flavoured()}/lib/python{self.version.flavoured()}/site-packages/pip_install_test'
assert(path.exists())
return Status.SUCCESS
def test_tkinter_usage(self):
'''Test basic tkinter usage'''
try:
os.environ['DISPLAY']
self.env['TK_LIBRARY']
except KeyError:
return Status.SKIPPED
else:
tkinter = 'tkinter' if self.version.major > 2 else 'Tkinter'
Script(f'''
import {tkinter} as tkinter
tkinter.Tk()
''').run(self.appimage)
return Status.SUCCESS
def test_venv_usage(self):
'''Test venv creation'''
if self.version.major == 2:
return Status.SKIPPED
else:
system(' && '.join((
f'cd {self.tmpdir.name}',
f'./{self.appimage.name} -m venv ENV',
'. ENV/bin/activate',
)))
python = Path(f'{self.tmpdir.name}/ENV/bin/python')
assert_eq(self.appimage.name, str(python.readlink()))
return Status.SUCCESS
def test():
'''Test Python AppImage(s)'''
for appimage in ARGS.appimage:
context = TestContext(appimage)
context.run()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description = test.__doc__)
parser.add_argument('appimage',
help = 'path to appimage(s)',
nargs = '+',
type = lambda x: Path(x).absolute()
)
ARGS = parser.parse_args()
test()

View File

@@ -16,9 +16,9 @@ from python_appimage.utils.manylinux import format_appimage_name, format_tag
# Build matrix
ARCHS = ('x86_64', 'i686')
ARCHS = ('x86_64', 'i686', 'aarch64')
MANYLINUSES = ('1', '2010', '2014', '2_24', '2_28')
EXCLUDES = ('2_28_i686',)
EXCLUDES = ('2_28_i686', '1_aarch64', '2010_aarch64')
# Build directory for AppImages
APPIMAGES_DIR = 'build-appimages'
@@ -202,7 +202,8 @@ def update(args):
if new_assets:
log('DRY', f'new update summary with {len(new_assets)} entries')
return
if not args.build:
return
if new_assets:
# Build new AppImage(s)
@@ -215,6 +216,9 @@ def update(args):
finally:
os.chdir(cwd)
if args.dry:
return
# Create any new release(s).
for tag in new_releases:
meta = ReleaseMeta(tag)
@@ -305,6 +309,11 @@ if __name__ == '__main__':
action = 'store_true',
default = False
)
parser.add_argument('-b', '--build',
help = 'build AppImages (in dry mode)',
action = 'store_true',
default = False
)
parser.add_argument('-d', '--dry',
help = 'dry run (only log changes)',
action = 'store_true',

View File

@@ -1,88 +0,0 @@
import json
import os
import setuptools
import ssl
import subprocess
from python_appimage.utils.deps import ensure_excludelist
from python_appimage.utils.url import urlopen
CLASSIFIERS = '''\
Development Status :: 4 - Beta
Intended Audience :: Developers
License :: OSI Approved :: GNU General Public License v3 (GPLv3)
Programming Language :: Python
Topic :: Software Development
Operating System :: POSIX :: Linux
'''
with open('README.md') as f:
long_description = f.read()
def get_version():
'''Get the next version number from PyPI
'''
with open('VERSION') as f:
version = f.read().strip()
p = subprocess.Popen(
'git describe --match=NeVeRmAtCh --always --dirty 2> /dev/null || '
'echo unknown',
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, _ = p.communicate()
try:
stdout = stdout.decode()
except AttributeError:
stdout = str(stdout)
git_revision = stdout.strip()
with open('python_appimage/version.py', 'w+') as f:
f.write('''\
# This file was generated by setup.py
version = '{version:}'
git_revision = '{git_revision:}'
'''.format(version=version, git_revision=git_revision))
return version
def get_package_data():
'''Get the list of package data
'''
ensure_excludelist()
prefix = os.path.dirname(__file__) or '.'
return ['data/' + file_
for file_ in os.listdir(prefix + '/python_appimage/data')]
setuptools.setup(
name = 'python_appimage',
version = get_version(),
author = 'Valentin Niess',
author_email = 'valentin.niess@gmail.com',
description = 'Appimage releases of Python',
long_description = long_description,
long_description_content_type = 'text/markdown',
url = 'https://github.com/niess/python-appimage',
download_url = 'https://pypi.python.org/pypi/python-appimage',
project_urls = {
'Bug Tracker' : 'https://github.com/niess/python-appimage/issues',
'Source Code' : 'https://github.com/niess/python-appimage',
},
packages = setuptools.find_packages(),
classifiers = [s for s in CLASSIFIERS.split(os.linesep) if s.strip()],
license = 'GPLv3',
platforms = ['Linux'],
python_requires = '>=2.7',
include_package_data = True,
package_data = {'': get_package_data()},
entry_points = {
'console_scripts' : (
'python-appimage = python_appimage.__main__:main',)
}
)