Add JOSS paper

.github/workflows/appimage.yml

@@ -1,36 +1,89 @@
 name: AppImage
 on:
-  workflow_dispatch:
+  push:
-    inputs:
+    branches:
-      dry:
+      - master
-        description: 'Dry run'
+    paths:
-        required: true
+      - '.github/workflows/appimage.yml'
-        type: boolean
+      - 'python_appimage/**'
-      all:
-        description: 'Update all'
-        required: true
-        type: boolean
   schedule:
     - cron: '0 3 * * 0'
 
 jobs:
-  Update:
+  Build:
     runs-on: ubuntu-latest
-    permissions:
+    strategy:
-      contents: write
+      matrix:
+        image: ['1', '2010', '2014']
+        arch: [x86_64, i686]
+        tag: [cp27-cp27m, cp27-cp27mu, cp35-cp35m, cp36-cp36m, cp37-cp37m,
+              cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311]
+        exclude:
+          - image: '1'
+            tag: cp310-cp310
+          - image: '1'
+            tag: cp311-cp311
+          - image: '2010'
+            tag: cp27-cp27m
+          - image: '2010'
+            tag: cp27-cp27mu
+          - image: '2010'
+            tag: cp35-cp35m
+          - image: '2014'
+            tag: cp27-cp27m
+          - image: '2014'
+            tag: cp27-cp27mu
+          - image: '2014'
+            tag: cp35-cp35m
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
 
-    - name: Install Dependencies
+    - name: Build
-      run: pip install PyGithub
-
-    - name: Run updater
-      run: |
-        ./scripts/update-appimages.py \
-          --token=${{ secrets.GITHUB_TOKEN }} \
-          --sha=${{ github.sha }} \
-          ${{ inputs.all && '--all' || '' }} \
-          ${{ inputs.dry && '--dry' || '' }}
       env:
-        PYTHONPATH: ${{ github.workspace }}
+        ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
+      run: |
+        # Build the AppImage
+        python -m python_appimage build manylinux \
+          ${{ matrix.image }}_${{ matrix.arch }} \
+          ${{ matrix.tag }}
+
+        # Export the AppImage name and the Python version
+        appimage=$(ls python*.AppImage)
+        SCRIPT=$(cat <<-END
+        version = '${appimage}'[6:].split('.', 2)
+        print('{:}.{:}'.format(*version[:2]))
+        END
+        )
+        version=$(python -c "${SCRIPT}")
+
+        echo "::set-env name=PYTHON_APPIMAGE::${appimage}"
+        echo "::set-env name=PYTHON_VERSION::${version}"
+
+    - uses: actions/upload-artifact@v1
+      if: github.ref == 'refs/heads/master'
+      with:
+        name: python${{ env.PYTHON_VERSION }}-appimages
+        path: ${{ env.PYTHON_APPIMAGE }}
+
+  Release:
+    needs: Build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
+    strategy:
+      matrix:
+        version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/download-artifact@v1
+      with:
+        name: python${{ matrix.version }}-appimages
+
+    - name: Release
+      uses: marvinpinto/action-automatic-releases@latest
+      with:
+        automatic_release_tag: python${{ matrix.version }}
+        title: Python ${{ matrix.version }}
+        files: |
+          python${{ matrix.version }}-appimages/python*.AppImage
+        repo_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/applications.yml

@@ -1,39 +1,27 @@
 name: Applications
 on:
-  workflow_dispatch:
+  push:
-    inputs:
+    branches:
-      scipy:
+      - master
-        required: true
+    paths:
-        default: true
+      - '.github/workflows/applications.yml'
-        type: boolean
+      - 'applications/**'
-      tasmotizer:
+      - 'python_appimage/**'
-        required: true
-        default: true
-        type: boolean
-      xonsh:
-        required: true
-        default: true
-        type: boolean
-      ssh-mitm:
-        required: true
-        default: true
-        type: boolean
 
 jobs:
   Test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         version: ['2.7', '3.7', '3.9']
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.version }}
 
     - name: Test scipy
-      if: ${{ inputs.scipy }}
       run: |
         python -m python_appimage build app applications/scipy                 \
                                             --python-version=2.7               \
@@ -42,22 +30,19 @@ jobs:
         ./scipy-x86_64.AppImage -c 'import numpy, pandas, scipy'
 
     - name: Test tasmotizer
-      if: ${{ inputs.tasmotizer }}
       run: |
         python -m python_appimage build app applications/tasmotizer            \
-                                            --linux-tag=manylinux1_x86_64      \
+                                            --linux-tag=manylinux2014_x86_64
-                                            --python-version=3.9
         test -e tasmotizer-x86_64.AppImage
 
     - name: Test xonsh
-      if: ${{ inputs.xonsh }}
       run: |
         python -m python_appimage build app applications/xonsh
         test -e xonsh-x86_64.AppImage
         ./xonsh-x86_64.AppImage -c 'import xonsh'
 
     - name: Test ssh-mitm
-      if: ${{ inputs.ssh_mitm && (matrix.version == '3.9') }}
+      if: ${{ matrix.version == '3.9' }}
       run: |
         python -m python_appimage build app applications/ssh-mitm
         test -e ssh-mitm-x86_64.AppImage

.github/workflows/delete-artifacts.yml

@@ -0,0 +1,13 @@
+name: Delete artifacts
+on:
+  schedule:
+    - cron: '0 3 * * 0'
+
+jobs:
+  delete-artifacts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: kolpav/purge-artifacts-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          expire-in: 0days

.github/workflows/pypi.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   Test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         version: ['2.7', '3.9']

README.md

@@ -31,22 +31,11 @@ Python apps, given an existing Python AppImage and a recipe folder.
   through the ssh
 
 
-## License
-
-The [`python-appimage`][PYPI] package (**A**) is under the GNU GPLv3 license,
-except for files located under `python_appimage/data` which are MIT licensed.
-Thus, the produced Manylinux Python AppImages (**B**) are not GPL'd. They
-contain a CPython distribution that is (mostly) under the [PSF
-license][PSF_LICENSE]. Other parts of **B** (e.g. `AppRun`) are under the MIT
-license.
-
-
 [APPLICATIONS]: https://github.com/niess/python-appimage/tree/master/applications
 [APPIMAGE]: https://appimage.org/
 [GITHUB]: https://github.com/niess/python-appimage
 [GRAND]: http://grand.cnrs.fr
 [MANYLINUX]: https://github.com/pypa/manylinux
-[PSF_LICENSE]: https://docs.python.org/3/license.html#psf-license
 [PYPI]: https://pypi.org/project/python-appimage/
 [READTHEDOCS]: https://python-appimage.readthedocs.io/en/latest/
 [RELEASES]: https://github.com/niess/python-appimage/releases

VERSION

@@ -1 +1 @@
-1.2.5
+1.1.2

docs/include/references.md

@@ -13,7 +13,7 @@
 [LINUXDEPLOY]: https://github.com/linuxdeploy/linuxdeploy/
 [MANYLINUX]: https://github.com/pypa/manylinux/
 [PATCHELF]: https://github.com/NixOS/patchelf/
-[PYPI]: https://pypi.org/project/python-appimage/
+[PYPI]: https://pypi.org/project/python-appimaAge/
 [RELEASES]: {{ config.repo_url }}releases/
 [SHEBANG]: https://en.wikipedia.org/wiki/Shebang_(Unix)/
 [VENV]: https://docs.python.org/3/library/venv.html/

docs/src/apps.md

@@ -121,9 +121,9 @@ files. The `requirements.txt` file allows to specify additional site packages
 to be bundled in the AppImage, using `pip`.
 
 !!! Caution
-    For the application to be portable, site packages bundled in the AppImage,
+    Site packages bundled in the AppImage, as well as their dependencies, must
-    as well as their dependencies, must must be available as binary wheels, or
+    either be pure python packages, or they must be available as portable binary
-    be pure Python packages.
+    wheels.
 
     If a **C extension** is bundled from **source**, then it will likely **not
     be portable**, as further discussed in the [Advanced
@@ -134,13 +134,6 @@ to be bundled in the AppImage, using `pip`.
     be cross-checked by browsing the `Download files` section on the package's
     PyPI page.
 
-!!! Tip
-    Since version 1.2, `python-appimage` allows to specify local requirements as
-    well, using the `local+` tag (see
-    [PR49](https://github.com/niess/python-appimage/pull/49)). Note however that
-    this performs a direct copy of the local package, which has several
-    limitations.
-
 {{ begin(".capsule") }}
 ### Entry point script
 

docs/src/js/index.js

@@ -11,23 +11,11 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
                 /* Parse AppImage metadata */
                 const tmp0 = asset.name.split("manylinux")
                 const tag = tmp0[1].slice(0,-9);
-                const tmp1 = tag.split(/_(.+)/, 2);
+                const tmp1 = tag.split(/_(.+)/);
-                var linux = undefined;
+                const linux = tmp1[0]
-                var arch = undefined;
+                const arch = tmp1[1]
-                if (tmp1[0] == "") {
+                const tmp2 = tmp0[0].split("-")
-                    const tmp3 = tmp1[1].split("_");
+                const python = tmp2[1] + "-" + tmp2[2]
-                    linux = tmp3[0] + "_" + tmp3[1];
-                    if (tmp3.length == 3) {
-                        arch = tmp3[2];
-                    } else {
-                        arch = tmp3[2] + "_" + tmp3[3];
-                    }
-                } else {
-                    linux = tmp1[0];
-                    arch = tmp1[1];
-                }
-                const tmp2 = tmp0[0].split("-", 3);
-                const python = tmp2[1] + "-" + tmp2[2];
                 assets.push({
                     name: asset.name,
                     url: asset.browser_download_url,
@@ -37,8 +25,8 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
                 });
 
                 if (full_version === undefined) {
-                    const index = asset.name.indexOf("-");
+                    const index = asset.name.indexOf("-")
-                    full_version = asset.name.slice(6, index);
+                    full_version = asset.name.slice(6, index)
                 }
             }
         }
@@ -174,17 +162,6 @@ $.getJSON("https://api.github.com/repos/niess/python-appimage/releases").done(fu
 
             ln -s ${appdir}/AppRun python${release.version}
         `);
-
-        set_snippet("#repackaging-example", `\
-            wget https://github.com/AppImage/AppImageKit/releases/download/continuous/\\
-            appimagetool-x86_64.AppImage
-
-            chmod +x appimagetool-x86_64.AppImage
-
-            ./appimagetool-x86_64.AppImage \\
-                ${appdir} \\
-                ${asset.name}
-        `);
     }
 
 

paper/Makefile

@@ -0,0 +1,13 @@
+paper.pdf: paper.md paper.bib
+	docker run --rm \
+	           --volume ${PWD}:/data \
+	           --user $(id -u):$(id -g) \
+	           --env JOURNAL=joss \
+	           openjournals/inara \
+	           -o pdf \
+	           /data/paper.md
+
+
+.PHONY: clean
+clean:
+	rm -f paper.pdf

paper/paper.bib

@@ -0,0 +1,132 @@
+@software{kernel,
+        title        = {Linux Kernel},
+        author       = "Torvalds, Linus and others",
+        howpublished = "\url{https://kernel.org/}",
+        year         = 1991
+}
+
+@software{appimage,
+        title        = {AppImage},
+        booktitle    = {Linux applications that run everywhere},
+        author       = "Peter, Simon and others",
+        howpublished = "\url{https://appimage.org/}",
+        year         = 2013
+}
+
+@software{python,
+        title        = {Python},
+        author       = "{van Rossum}, Guido and others",
+        howpublished = "\url{https://www.python.org/}",
+        year         = 1991
+}
+
+@software{anaconda,
+        title        = {Anaconda},
+        author       = "Anaconda{, Inc.}",
+        howpublished = "\url{https://www.anaconda.com/}",
+        year         = 2012
+}
+
+@software{wheels,
+        title        = {Python wheels},
+        author       = "{Python Software Foundation}",
+        howpublished = "\url{https://pythonwheels.com/}",
+        year         = 2012
+}
+
+@software{flatpak,
+        title        = {Flatpak},
+        author       = "Larsson, Alexander and others",
+        howpublished = "\url{https://flatpak.org/}",
+        year         = 2015
+}
+
+@software{snap,
+        title        = {Snap},
+        author       = "{Canonical Group Limited}",
+        howpublished = "\url{https://snapcraft.io/}",
+        year         = 2016
+}
+
+@software{libfuse,
+        title        = {Linux FUSE},
+        author       = "Szeredi, Miklos and Rath, Nikolaus and others",
+        howpublished = "\url{https://github.com/libfuse/libfuse}",
+        year         = 2013
+}
+
+@software{glibc,
+        title        = {The GNU C Library},
+        author       = "McGrath, Roland and Drepper, Ulrich and others",
+        howpublished = "\url{https://www.gnu.org/software/libc/}",
+        year         = 1987
+}
+
+@software{docker,
+        title        = {Docker},
+        author       = "{Docker, Inc.}",
+        howpublished = "\url{https://www.docker.com/}",
+        year         = 2013
+}
+
+@software{singularity,
+        title        = {Singularity},
+        author       = "Kurtzer, Gregory and others",
+        howpublished = "\url{https://apptainer.org/}",
+        year         = 2015
+}
+
+@software{patchelf,
+        title        = {PatchELF},
+        author       = "Dolstra, Eelco and others",
+        howpublished = "\url{https://github.com/NixOS/patchelf}",
+        year         = 2004
+}
+
+@software{linuxdeploy,
+        title        = {linuxdeploy},
+        author       = "{@TheAssassin} and others",
+        howpublished = "\url{https://github.com/linuxdeploy/linuxdeploy}",
+        year         = 2018
+}
+
+@online{Gillmor:2014,
+        title        = "{Q}\&{A} with {Linus} {Torvalds}",
+        date         = 2014,
+        organization = "{Debian} {Conference} 2014",
+        author       = "Gillmor, Daniel and Guerrero López, Ana and Torvalds,
+                        Linus and others",
+        url          = {https://www.youtube.com/watch?v=5PmHRSeA2c8}
+}
+
+@online{manylinux,
+        title        = "Manylinux",
+        date         = 2016,
+        organization = "{GitHub}",
+        author       = "{Python Packaging Authority}",
+        url          = {https://github.com/pypa/manylinux}
+}
+
+@online{tiobe,
+        title        = "TIOBE index",
+        date         = 2022,
+        author       = "{The Software Quality Company}",
+        url          = {https://www.tiobe.com/tiobe-index/}
+}
+
+@online{grand,
+        title        = "Giant Radio Array for Neutrino Detection",
+        date         = 2018,
+        author       = "{The GRAND collaboration}",
+        url          = {https://grand.cnrs.fr/}
+}
+
+@article{Alvarez-Muniz:2020,
+   author  = "J. Álvarez-Muñiz and others",
+   doi     = {10.1007/s11433-018-9385-7},
+   issue   = {1},
+   journal = {Science China: Physics, Mechanics and Astronomy},
+   title   = {The Giant Radio Array for Neutrino Detection (GRAND): Science and design},
+   volume  = {63},
+   year    = {2020}
+}

paper/paper.md

@@ -0,0 +1,150 @@
+---
+title: 'The Python-AppImage project'
+tags:
+  - AppImage
+  - Linux
+  - Packaging
+  - Python
+authors:
+  - name: Valentin Niess
+    orcid: 0000-0001-7148-6819
+    affiliation: 1
+affiliations:
+ - name: Université Clermont Auvergne, CNRS/IN2P3, LPC, F-63000 Clermont-Ferrand, France.
+   index: 1
+date: 22 June 2022
+bibliography: paper.bib
+---
+
+# Summary
+
+Since its initial release in 1991, the Linux Kernel [@kernel] has given birth to
+more than 600 hundred Linux distributions (distros). While this is an impressive
+success, the diversity of Linux flavours complicates the distribution of
+applications for Linux (see e.g. [@Gillmor:2014]). Thus, contrary to other
+operating systems, on Linux, source distributions long prevailed over binary
+(precompiled) ones. Specifically, this is still the case for the Linux Python
+runtime [@python] distributed by the Python Software Foundation (PSF).
+Previously, this was also the case for Python packages available from the Python
+Package Index (PyPI). This situation contributed to the emergence of an
+alternative Python packaging system [@anaconda] delivering ready-to-use
+precompiled runtimes and packages for Linux.
+
+Over the last decade, a change of paradigm occurred in the Linux world. Cross
+distros packaging systems have emerged, the like AppImage [@appimage], Flatpak
+[@flatpak] and Snap [@snap]. At the same time, the PSF encouraged the conversion
+of PyPI packages from source to binary distributions, using the new `wheel`
+packaging format [@wheels].
+
+The AppImage format is of particular interest for the present discussion.
+Contrary to Flatpak and Snap, AppImages do not require to install an external
+package manager. AppImage applications are bundled as a single executable file
+upstreamed by developpers, ready-to-use after download. However, building proper
+AppImages adds some complexity on the developers side.
+
+Technically, an AppImage is an executable file embedding a software application
+over a compressed virtual filesystem (VFS). The VFS is extracted and mounted at
+runtime using `libfuse` [@libfuse]. Apart from core libraries, the like `libc`,
+application dependencies are directly bundled inside the AppImage. In this
+context, binary compatibility usually stems down to the host glibc [@glibc]
+version used at compile time. As a matter of fact, glibc is admirably backward
+compatible, down to version 2.1 where symbol versioning was introduced. Thus, in
+practice binary compatibility is achieved by precompiling the application and
+its relevant dependencies on an *old enough* Linux distro. This is greatly
+facilitated by recent container technologies, the like Docker [@docker] and
+Singularity [@singularity].
+
+In practice, producing a portable binary wheel for Linux, or packaging an
+application as an AppImage, faces the same issues. Accordingly, almost identical
+strategies and tools are used in both cases. In particular, the PSF has
+defined standard build platforms for Linux wheels. Those are available as Docker
+images from the Manylinux project [@manylinux]. These images contain
+precompiled Python runtimes with binary compatibility down to glibc 2.5 (for
+manylinux1, corresponding to CentOS 5).
+
+The Python-AppImage project provides relocatable Python runtimes as AppImages.
+These runtimes are extracted from the Manylinux Docker images. Consequently,
+they have the exact same binary compatibility as PyPI wheels. Python AppImages
+are available as rolling GitHub releases. They are updated automatically on
+every Sunday using GitHub Actions.
+
+At the core of the Python-AppImage project, there is the `python-appimage`
+executable, written in pure Python, and available from PyPI using `pip install`
+(down to Python 2.7). The main functionality of `python-appimage` is to relocate
+an existing (system) Python installation to an AppImage. Vanilla Python is not
+relocatable. Therefore, some tweaks are needed in order to run Python from an
+AppImage. In particular,
+
+- Python initialisation is slightly modified in order to set `sys.executable`
+  and `sys.prefix` to the AppImage and to its temporary mount point. This is
+  achieved by a small edit of the `site` package.
+
+- The run-time search path of Linux ELF files (Python runtime, binary packages,
+  shared libraries) is changed to a relative location, according to the AppImage
+  VFS hierarchy. Note that `patchelf` [@patchelf] allows to edit the
+  corresponding ELF entry. Thus, the Python runtime and its binary packages need
+  not to be recompiled.
+
+Besides, `python-appimage` can also build simple Python based applications
+according to a recipe folder. The recipe folder contains an entry point script,
+AppImage metadata and an optional `requirements.txt` Python file. The
+application is built from an existing  Manylinux Python AppImage. Extra
+dependencies are fetched from PyPI with `pip`. Note that they must be available
+as binary wheels for the resulting AppImage to be portable.
+
+
+# Statement of need
+
+Python is among the top computing languages used nowadays. It was ranked number
+one in 2021 and 2022 according to the TIOBE index [@tiobe]. In particular,
+Python is widely used by the scientific community. A peculiarity of the Python
+language is that it constantly evolves, owing to an Open Source community of
+contributors structured by the PSF. While I personally find this very exciting
+when working with Python, it comes at a price. Different projects use different
+Python versions, and package requirements might conflict. This can be
+circumvented by using virtual environments. For example, the `venv` package
+allows one to manage different sets of requirements. However, it requires an
+existing Python installation, i.e. a specific runtime version. On the contrary,
+`conda` environments automate the management of both different runtime versions,
+and sets of requirements. But, unfortunately Anaconda fragmented Python
+packaging since, in practice, `conda` packages and Python wheels are not
+(always) binary compatible.
+
+Python-AppImage offers alternative solutions when working within the PSF context
+(for `conda` based AppImages, `linuxdeploy` [@linuxdeploy] could be used).
+First, Python-AppImage complements `venv` by providing ready-to-use Linux
+runtimes for different Python versions. Moreover, Python AppImages can be used
+as a replacement to `venv` by bundling extra (site) packages aside the runtime.
+In this case, since AppImages are read-only, it can be convenient to extract
+their content to a local folder (using the built-in `--appimage-extract`
+option). Then, the extracted AppImage appears as a classic Python installation,
+but it is relocatable. Especially, one can `pip install` additional packages to
+the extracted folder, and it can be moved around, e.g. from a development host
+directly to a production computing center.
+
+
+# Mention
+
+This project was born in the context of the Giant Radio Array for Neutrino
+Detection (GRAND) [@grand] in order to share Python based software across
+various Linux hosts: personal or office computers, computing centers, etc.
+Specifically, Python AppImages were used for the neutrino sensitivity studies
+presented in the GRAND whitepaper [@Alvarez-Muniz:2020].
+
+I do not track usage of Python AppImages, which are available as simple
+downloads. Therefore, I cannot provide detailed statistics. However, I am aware
+of a dozen of (non scientific) Python based applications using
+`python-appimage`. Besides, I personally use Python AppImages for my daily
+academic work.
+
+
+# Acknowledgements
+
+We are grateful to several GitHub users for comments, feedback and contributions
+to this project. In particular, we would like to thank Andy Kipp (\@anki-code),
+Simon Peter (\@probonopd) and \@TheAssassin for support at the very roots of
+this project. In addition, we are grateful to our colleagues from the GRAND
+collaboration for beta testing Python AppImages.
+
+
+# References

python_appimage/__main__.py

@@ -1,5 +1,6 @@
 import argparse
 from importlib import import_module
+import logging
 import os
 import sys
 
@@ -23,9 +24,9 @@ def main():
                                        dest='command')
 
     parser.add_argument('-q', '--quiet', help='disable logging',
-        dest='verbosity', action='store_const', const='ERROR')
+        dest='verbosity', action='store_const', const=logging.ERROR)
     parser.add_argument('-v', '--verbose', help='print extra information',
-        dest='verbosity', action='store_const', const='DEBUG')
+        dest='verbosity', action='store_const', const=logging.DEBUG)
 
     install_parser = subparsers.add_parser('install',
         description='Install binary dependencies')
@@ -74,11 +75,6 @@ def main():
                                   action='store_true',
                                   default=False)
 
-    list_parser = subparsers.add_parser('list',
-        description='List Python versions installed in a manylinux image')
-    list_parser.add_argument('tag',
-        help='manylinux image tag (e.g. 2010_x86_64)')
-
     which_parser = subparsers.add_parser('which',
         description='Locate a binary dependency')
     which_parser.add_argument('binary', choices=binaries,
@@ -88,8 +84,7 @@ def main():
 
     # Configure the verbosity
     if args.verbosity:
-        from .utils import log
+        logging.getLogger().setLevel(args.verbosity)
-        log.set_level(args.verbosity)
 
     # check if no arguments are passed
     if args.command is None:

python_appimage/appimage/relocate.py

@@ -34,15 +34,6 @@ def _get_tk_version(python_pkg):
             raise RuntimeError('could not guess Tcl/Tk version')
 
 
-def _get_tk_libdir(version):
-    try:
-        library = system(('tclsh' + version,), stdin='puts [info library]')
-    except SystemError:
-        raise RuntimeError('could not locate Tcl/Tk' + version + ' library')
-
-    return os.path.dirname(library)
-
-
 def tcltk_env_string(python_pkg):
     '''Environment for using AppImage's TCl/Tk
     '''
@@ -98,7 +89,7 @@ def patch_binary(path, libdir, recursive=True):
     rpath = '\'' + system((PATCHELF, '--print-rpath', path)) + '\''
     relpath = os.path.relpath(libdir, os.path.dirname(path))
     relpath = '' if relpath == '.' else '/' + relpath
-    expected = '\'$ORIGIN' + relpath + ':$ORIGIN/../lib\''
+    expected = '\'$ORIGIN' + relpath + '\''
     if rpath != expected:
         system((PATCHELF, '--set-rpath', expected, path))
 
@@ -289,13 +280,22 @@ def relocate_python(python=None, appdir=None):
         tcltkdir = APPDIR_SHARE + '/tcltk'
         if (not os.path.exists(tcltkdir + '/tcl' + tk_version)) or             \
            (not os.path.exists(tcltkdir + '/tk' + tk_version)):
-            libdir = _get_tk_libdir(tk_version)
+            hostdir = '/usr/share/tcltk'
-            log('INSTALL', 'Tcl/Tk' + tk_version)
+            if os.path.exists(hostdir):
-            make_tree(tcltkdir)
+                make_tree(APPDIR_SHARE)
-            tclpath = libdir + '/tcl' + tk_version
+                copy_tree(hostdir, tcltkdir)
-            copy_tree(tclpath, tcltkdir + '/tcl' + tk_version)
+            else:
-            tkpath = libdir + '/tk' + tk_version
+                make_tree(tcltkdir)
-            copy_tree(tkpath, tcltkdir + '/tk' + tk_version)
+                tclpath = '/usr/share/tcl' + tk_version
+                if not tclpath:
+                    raise ValueError('could not find ' + tclpath)
+                copy_tree(tclpath, tcltkdir + '/tcl' + tk_version)
+
+                tkpath = '/usr/share/tk' + tk_version
+                if not tkpath:
+                    raise ValueError('could not find ' + tkpath)
+                copy_tree(tkpath, tcltkdir + '/tk' + tk_version)
+
 
     # Copy any SSL certificate
     cert_file = os.getenv('SSL_CERT_FILE')

python_appimage/commands/build/app.py

@@ -8,9 +8,9 @@ import stat
 import struct
 
 from ...appimage import build_appimage
-from ...utils.compat import decode, find_spec
+from ...utils.compat import decode
 from ...utils.deps import PREFIX
-from ...utils.fs import copy_file, copy_tree, make_tree, remove_file, remove_tree
+from ...utils.fs import copy_file, make_tree, remove_file, remove_tree
 from ...utils.log import log
 from ...utils.system import system
 from ...utils.template import copy_template, load_template
@@ -253,31 +253,15 @@ def execute(appdir, name=None, python_version=None, linux_tag=None,
                 'WARNING: Running pip as'
             )
 
-            isolation_flag = '-sE' if python_version[0] == '2' else '-I'
+            system(('./AppDir/AppRun', '-m', 'pip', 'install', '-U', in_tree_build,
-            system(('./AppDir/AppRun', isolation_flag, '-m', 'pip', 'install', '-U', in_tree_build,
                    '--no-warn-script-location', 'pip'), exclude=deprecation)
             for requirement in requirements_list:
                 if requirement.startswith('git+'):
                     url, name = os.path.split(requirement)
                     log('BUNDLE', name + ' from ' + url[4:])
-                elif requirement.startswith('local+'):
-                    name = requirement[6:]
-                    source = find_spec(name).origin
-                    if source.endswith('/__init__.py'):
-                        source = os.path.dirname(source)
-                    elif source.endswith('/'):
-                        source = source[:-1]
-                    log('BUNDLE', name + ' from ' + source)
-                    if os.path.isfile(source):
-                        destination = 'AppDir/opt/python{0:}/lib/python{0:}/site-packages/'.format(python_version)
-                        copy_file(source, destination)
-                    else:
-                        destination = 'AppDir/opt/python{0:}/lib/python{0:}/site-packages/{1:}'.format(python_version, name)
-                        copy_tree(source, destination)
-                    continue
                 else:
                     log('BUNDLE', requirement)
-                system(('./AppDir/AppRun', isolation_flag, '-m', 'pip', 'install', '-U', in_tree_build,
+                system(('./AppDir/AppRun', '-m', 'pip', 'install', '-U', in_tree_build,
                        '--no-warn-script-location', requirement),
                        exclude=(deprecation, '  Running command git clone'))
 

python_appimage/commands/build/manylinux.py

@@ -7,7 +7,6 @@ import sys
 from ...appimage import build_appimage, relocate_python
 from ...utils.docker import docker_run
 from ...utils.fs import copy_tree
-from ...utils.manylinux import format_appimage_name, format_tag
 from ...utils.tmp import TemporaryDirectory
 
 
@@ -28,7 +27,8 @@ def _get_appimage_name(abi, tag):
     fullversion = desktop[13:-8]
 
     # Finish building the AppImage on the host. See below.
-    return format_appimage_name(abi, fullversion, tag)
+    return 'python{:}-{:}-manylinux{:}.AppImage'.format(
+        fullversion, abi, tag)
 
 
 def execute(tag, abi, contained=False):
@@ -37,7 +37,7 @@ def execute(tag, abi, contained=False):
 
     if not contained:
         # Forward the build to a Docker image
-        image = 'quay.io/pypa/' + format_tag(tag)
+        image = 'quay.io/pypa/manylinux' + tag
         python = '/opt/python/' + abi + '/bin/python'
 
         pwd = os.getcwd()
@@ -45,11 +45,7 @@ def execute(tag, abi, contained=False):
         with TemporaryDirectory() as tmpdir:
             copy_tree(dirname, 'python_appimage')
 
-            argv = sys.argv[1:]
+            argv = ' '.join(sys.argv[1:])
-            if argv:
-                argv = ' '.join(argv)
-            else:
-                argv = 'build manylinux {:} {:}'.format(tag, abi)
             if tag.startswith("1_"):
                 # On manylinux1 tk is not installed
                 script = [

python_appimage/commands/list.py

@@ -1,43 +0,0 @@
-import os
-
-from ..utils.docker import docker_run
-from ..utils.log import log
-from ..utils.tmp import TemporaryDirectory
-
-
-__all__ = ['execute']
-
-
-def _unpack_args(args):
-    '''Unpack command line arguments
-    '''
-    return (args.tag,)
-
-
-def execute(tag):
-    '''List python versions installed in a manylinux image
-    '''
-
-    with TemporaryDirectory() as tmpdir:
-        script = (
-            'for dir in $(ls /opt/python | grep "^cp[0-9]"); do',
-            '   version=$(/opt/python/$dir/bin/python -c "import sys; ' \
-                    'sys.stdout.write(sys.version.split()[0])")',
-            '   echo "$dir $version"',
-            'done',
-        )
-        if tag.startswith('2_'):
-            image = 'manylinux_' + tag
-        else:
-            image = 'manylinux' + tag
-        result = docker_run(
-            'quay.io/pypa/' + image,
-            script,
-            capture = True
-        )
-        pythons = [line.split() for line in result.split(os.linesep) if line]
-
-        for (abi, version) in pythons:
-            log('LIST', "{:7} ->  /opt/python/{:}".format(version, abi))
-
-        return pythons

python_appimage/data/LICENSE

@@ -1,19 +0,0 @@
-Copyright (c) Université Clermont Auvergne, CNRS/IN2P3, LPC
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

python_appimage/data/python-wrapper.sh

@@ -2,16 +2,16 @@
 
 # If running from an extracted image, then export ARGV0 and APPDIR
 if [ -z "${APPIMAGE}" ]; then
-    export ARGV0="$0"
+    export ARGV0=$0
 
-    self=$(readlink -f -- "$0") # Protect spaces (issue 55)
+    self="$(readlink -f -- $0)"
     here="${self%/*}"
     tmp="${here%/*}"
     export APPDIR="${tmp%/*}"
 fi
 
 # Resolve the calling command (preserving symbolic links).
-export APPIMAGE_COMMAND=$(command -v -- "$ARGV0")
+export APPIMAGE_COMMAND="$(command -v -- $ARGV0)"
 {{ tcltk-env }}
 {{ cert-file }}
 

python_appimage/utils/compat.py

@@ -1,7 +1,4 @@
-import sys
+__all__ = ['decode']
-
-
-__all__ = ['decode', 'encode', 'find_spec']
 
 
 def decode(s):
@@ -11,30 +8,3 @@ def decode(s):
         return s.decode()
     except Exception:
         return str(s)
-
-
-def encode(s):
-    '''Encode Python 3 str as bytes
-    '''
-    try:
-        return s.encode()
-    except Exception:
-        return str(s)
-
-
-if sys.version_info[0] == 2:
-    from collections import namedtuple
-    import imp
-
-    ModuleSpec = namedtuple('ModuleSpec', ('name', 'origin'))
-
-    def find_spec(name):
-        return ModuleSpec(name, imp.find_module(name)[1])
-
-else:
-    import importlib
-    try:
-        find_spec = importlib.util.find_spec
-    except AttributeError:
-        import importlib.util
-        find_spec = importlib.util.find_spec

python_appimage/utils/docker.py

@@ -4,12 +4,11 @@ import stat
 import subprocess
 import sys
 
-from .compat import decode
 from .log import log
 from .system import system
 
 
-def docker_run(image, extra_cmds, capture=False):
+def docker_run(image, extra_cmds):
     '''Execute commands within a docker container
     '''
 
@@ -43,16 +42,10 @@ def docker_run(image, extra_cmds, capture=False):
                     'type=bind,source={:},target=/pwd'.format(os.getcwd()),
                     image, '/bin/bash', bash_arg))
 
-    if capture:
-        opts = {'stderr': subprocess.PIPE, 'stdout': subprocess.PIPE}
-    else:
-        opts = {}
     log('RUN', image)
-    p = subprocess.Popen(cmd, shell=True, **opts)
+    p = subprocess.Popen(cmd, shell=True)
-    r = p.communicate()
+    p.communicate()
     if p.returncode != 0:
         if p.returncode == 139:
             sys.stderr.write("segmentation fault when running Docker (139)\n")
         sys.exit(p.returncode)
-    if capture:
-        return decode(r[0])

python_appimage/utils/log.py

@@ -7,25 +7,17 @@ __all__ = ['debug', 'log']
 # Configure the logger
 logging.basicConfig(
     format='[%(asctime)s] %(message)s',
-    level=logging.ERROR
+    level=logging.INFO
 )
-logging.getLogger('python-appimage').setLevel(logging.INFO)
 
 
 def log(task, fmt, *args):
     '''Log a standard message
     '''
-    logging.getLogger('python-appimage').info('%-8s ' + fmt, task, *args)
+    logging.info('%-8s ' + fmt, task, *args)
 
 
 def debug(task, fmt, *args):
     '''Report some debug information
     '''
-    logging.getLogger('python-appimage').debug('%-8s ' + fmt, task, *args)
+    logging.debug('%-8s ' + fmt, task, *args)
-
-
-def set_level(level):
-    '''Set the threshold for logs
-    '''
-    level = getattr(logging, level)
-    logging.getLogger('python-appimage').setLevel(level)

python_appimage/utils/manylinux.py

@@ -1,14 +0,0 @@
-def format_appimage_name(abi, version, tag):
-    '''Format the Python AppImage name using the ABI, python version and OS tags
-    '''
-    return 'python{:}-{:}-{:}.AppImage'.format(
-        version, abi, format_tag(tag))
-
-
-def format_tag(tag):
-    '''Format Manylinux tag
-    '''
-    if tag.startswith('2_'):
-        return 'manylinux_' + tag
-    else:
-        return 'manylinux' + tag

python_appimage/utils/system.py

@@ -2,8 +2,8 @@ import os
 import re
 import subprocess
 
-from .compat import decode, encode
+from .compat import decode
-from .log import debug, log
+from .log import debug
 
 
 __all__ = ['ldd', 'system']
@@ -15,7 +15,7 @@ except NameError:
     basestring = (str, bytes)
 
 
-def system(args, exclude=None, stdin=None):
+def system(args, exclude=None):
     '''System call with capturing output
     '''
     cmd = ' '.join(args)
@@ -29,15 +29,9 @@ def system(args, exclude=None, stdin=None):
         exclude = list(exclude)
     exclude.append('fuse: warning:')
 
-    if stdin:
-        in_arg = subprocess.PIPE
-        stdin = encode(stdin)
-    else:
-        in_arg = None
-
     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE, stdin=in_arg)
+                         stderr=subprocess.PIPE)
-    out, err = p.communicate(input=stdin)
+    out, err = p.communicate()
     if err:
         err = decode(err)
         stripped = [line for line in err.split(os.linesep) if line]
@@ -45,13 +39,7 @@ def system(args, exclude=None, stdin=None):
             stripped = [line for line in stripped
                         if not line.startswith(pattern)]
         if stripped:
-            # Tolerate single line warning(s)
+            raise RuntimeError(err)
-            for line in stripped:
-                if (len(line) < 8) or (line[:8].lower() != "warning:"):
-                    raise RuntimeError(err)
-            else:
-                for line in stripped:
-                    log('WARNING', line[8:].strip())
 
     return str(decode(out).strip())
 

scripts/update-appimages.py

@@ -1,312 +0,0 @@
-#! /usr/bin/env python3
-import argparse
-from collections import defaultdict
-from dataclasses import dataclass
-import os
-import subprocess
-import sys
-from typing import Optional
-
-from github import Auth, Github
-
-from python_appimage.commands.build.manylinux import execute as build_manylinux
-from python_appimage.commands.list import execute as list_pythons
-from python_appimage.utils.log import log
-from python_appimage.utils.manylinux import format_appimage_name, format_tag
-
-
-# Build matrix
-ARCHS = ('x86_64', 'i686')
-MANYLINUSES = ('1', '2010', '2014', '2_24', '2_28')
-EXCLUDES = ('2_28_i686',)
-
-# Build directory for AppImages
-APPIMAGES_DIR = 'build-appimages'
-
-
-@dataclass
-class ReleaseMeta:
-    '''Metadata relative to a GitHub release
-    '''
-
-    tag: str
-
-    ref: Optional["github.GitRef"] = None
-    release: Optional["github.GitRelease"] = None
-
-    def message(self):
-        '''Returns release message'''
-        return f'Appimage distributions of {self.title()} (see `Assets` below)'
-
-    def title(self):
-        '''Returns release title'''
-        version = self.tag[6:]
-        return f'Python {version}'
-
-
-@dataclass
-class AssetMeta:
-    '''Metadata relative to a release Asset
-    '''
-
-    tag: str
-    abi: str
-    version: str
-
-    asset: Optional["github.GitReleaseAsset"] = None
-
-    @classmethod
-    def from_appimage(cls, name):
-        '''Returns an instance from a Python AppImage name
-        '''
-        tmp = name[6:-9]
-        tmp, tag = tmp.split('-manylinux', 1)
-        if tag.startswith('_'):
-            tag = tag[1:]
-        version, abi = tmp.split('-', 1)
-        return cls(
-            tag = tag,
-            abi = abi,
-            version = version
-        )
-
-    def appimage_name(self):
-        '''Returns Python AppImage name'''
-        return format_appimage_name(self.abi, self.version, self.tag)
-
-    def formated_tag(self):
-        '''Returns formated manylinux tag'''
-        return format_tag(self.tag)
-
-    def previous_version(self):
-        '''Returns previous version'''
-        if self.asset:
-            return self.asset.name[6:-9].split('-', 1)[0]
-
-    def release_tag(self):
-        '''Returns release git tag'''
-        version = self.version.rsplit('.', 1)[0]
-        return f'python{version}'
-
-
-def update(args):
-    '''Update Python AppImage GitHub releases
-    '''
-
-    sha = args.sha
-    if sha is None:
-        sha = os.getenv('GITHUB_SHA')
-        if sha is None:
-            p = subprocess.run(
-                'git rev-parse HEAD',
-                shell = True,
-                capture_output = True,
-                check = True
-            )
-            sha = p.stdout.decode().strip()
-
-    # Connect to GitHub
-    token = args.token
-    if token is None:
-        # First, check for token in env
-        token = os.getenv('GITHUB_TOKEN')
-        if token is None:
-            # Else try to get a token from gh app
-            p = subprocess.run(
-                'gh auth token',
-                shell = True,
-                capture_output = True,
-                check = True
-            )
-            token = p.stdout.decode().strip()
-
-    auth = Auth.Token(token)
-    session = Github(auth=auth)
-    repo = session.get_repo('niess/python-appimage')
-
-    # Fetch currently released AppImages
-    log('FETCH', 'currently released AppImages')
-    releases = {}
-    assets = defaultdict(dict)
-    n_assets = 0
-    for release in repo.get_releases():
-        if release.tag_name.startswith('python'):
-            meta = ReleaseMeta(
-                tag = release.tag_name,
-                release = release
-            )
-            ref = repo.get_git_ref(f'tags/{meta.tag}')
-            if (ref.ref is not None) and (ref.object.sha != sha):
-                meta.ref = ref
-            releases[release.tag_name] = meta
-
-            for asset in release.get_assets():
-                if asset.name.endswith('.AppImage'):
-                    n_assets += 1
-                    meta = AssetMeta.from_appimage(asset.name)
-                    assert(meta.release_tag() == release.tag_name)
-                    meta.asset = asset
-                    assets[meta.tag][meta.abi] = meta
-
-    n_releases = len(releases)
-    log('FETCH', f'found {n_assets} AppImages in {n_releases} releases')
-
-    # Look for updates.
-    new_releases = set()
-    new_assets = []
-
-    for manylinux in MANYLINUSES:
-        for arch in ARCHS:
-            tag = f'{manylinux}_{arch}'
-            if tag in EXCLUDES:
-                continue
-
-            pythons = list_pythons(tag)
-            for (abi, version) in pythons:
-                try:
-                    meta = assets[tag][abi]
-                except KeyError:
-                    meta = None
-
-                if (meta is None) or (meta.version != version) or args.all:
-                    new_meta = AssetMeta(
-                        tag = tag,
-                        abi = abi,
-                        version = version
-                    )
-                    if meta is not None:
-                        new_meta.asset = meta.asset
-                    new_assets.append(new_meta)
-
-                    rtag = new_meta.release_tag()
-                    if rtag not in releases:
-                        new_releases.add(rtag)
-
-    if args.dry:
-        # Log foreseen changes and exit
-        for tag in new_releases:
-            meta = ReleaseMeta(tag)
-            log('DRY', f'new release for {meta.title()}')
-
-        for meta in new_assets:
-            log('DRY', f'create asset {meta.appimage_name()}')
-            if meta.asset is not None:
-                log('DRY', f'remove asset {meta.asset.name}')
-
-        for meta in releases.values():
-            if meta.ref is not None:
-                log('DRY', f'refs/tags/{meta.tag} -> {sha}')
-                if meta.release is not None:
-                    log('DRY', f'reformat release for {meta.title()}')
-
-        if new_assets:
-            log('DRY', f'new update summary with {len(new_assets)} entries')
-
-        return
-
-    if new_assets:
-        # Build new AppImage(s)
-        cwd = os.getcwd()
-        os.makedirs(APPIMAGES_DIR, exist_ok=True)
-        try:
-            os.chdir(APPIMAGES_DIR)
-            for meta in new_assets:
-                build_manylinux(meta.tag, meta.abi)
-        finally:
-            os.chdir(cwd)
-
-    # Create any new release(s).
-    for tag in new_releases:
-        meta = ReleaseMeta(tag)
-        title = meta.title()
-        meta.release = repo.create_git_release(
-            tag = meta.tag,
-            name = title,
-            message = meta.message(),
-            prerelease = True
-        )
-        releases[tag] = meta
-        log('UPDATE', f'new release for {title}')
-
-    # Update assets.
-    update_summary = []
-    for meta in new_assets:
-        release = releases[meta.release_tag()].release
-        appimage = meta.appimage_name()
-        new_asset = release.upload_asset(
-            path = f'{APPIMAGES_DIR}/{appimage}',
-            name = appimage
-        )
-        if meta.asset:
-            meta.asset.delete_asset()
-            update_summary.append(
-                f'- update {meta.formated_tag()}/{meta.abi} '
-                    f'{meta.previous_version()} -> {meta.version}'
-            )
-        else:
-            update_summary.append(
-                f'- add {meta.formated_tag()}/{meta.abi} {meta.version}'
-            )
-
-        meta.asset = new_asset
-        assets[meta.tag][meta.abi] = meta
-
-    # Update git tags SHA
-    for meta in releases.values():
-        if meta.ref is not None:
-            meta.ref.edit(
-                sha = sha,
-                force = True
-            )
-            log('UPDATE', f'refs/tags/{meta.tag} -> {sha}')
-
-            if meta.release is not None:
-                title = meta.title()
-                meta.release.update_release(
-                    name = title,
-                    message = meta.message(),
-                    prerelease = True,
-                    tag_name = meta.tag
-                )
-                log('UPDATE', f'reformat release for {title}')
-
-    # Generate update summary
-    if update_summary:
-        for release in repo.get_releases():
-            if release.tag_name == 'update-summary':
-                release.delete_release()
-                break
-
-        message = os.linesep.join(update_summary)
-        repo.create_git_release(
-            tag = 'update-summary',
-            name = 'Update summary',
-            message = message,
-            prerelease = True
-        )
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description = 'Update GitHub releases of Python AppImages'
-    )
-    parser.add_argument('-a', '--all',
-        help = 'force update of all available releases',
-        action = 'store_true',
-        default = False
-    )
-    parser.add_argument('-d', '--dry',
-        help = 'dry run (only log changes)',
-        action = 'store_true',
-        default = False
-    )
-    parser.add_argument("-s", "--sha",
-        help = "reference commit SHA"
-    )
-    parser.add_argument('-t', '--token',
-        help = 'GitHub authentication token'
-    )
-
-    args = parser.parse_args()
-    sys.argv = sys.argv[:1] # Empty args for fake call
-    update(args)