Update PyPI action

.github/workflows/pypi.yml

@@ -5,17 +5,23 @@ on:
       - master
     paths:
       - 'VERSION'
+  workflow_dispatch:
+    inputs:
+      upload:
+        description: 'Upload to PyPI'
+        required: true
+        type: boolean
 
 jobs:
   Test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        version: ['2.7', '3.9']
+        version: ['3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v1
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.version }}
 
@@ -31,10 +37,10 @@ jobs:
     if: github.ref == 'refs/heads/master'
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v1
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
 
     - name: Build wheel
       run: |
@@ -43,7 +49,7 @@ jobs:
         python setup.py bdist_wheel --universal
 
     - name: Upload to PyPI
-      if: github.ref == 'refs/heads/master'
+      if: (github.ref == 'refs/heads/master') && inputs.upload
       uses: pypa/gh-action-pypi-publish@master
       with:
         password: ${{ secrets.PYPI_TOKEN }}

VERSION

@@ -1 +1 @@
-1.2.5
+1.3.1

docs/src/apps.md

@@ -182,6 +182,36 @@ example, `$APPDIR` points to the AppImage mount point at runtime.
     `{{ python-executable }} -I` starts a fully isolated Python instance.
 {% endraw %}
 
+### Bundling data files
+
+`python-appimage` is also capable of bundling in auxilliary data files directly
+into the resulting AppImage. `-x/--extra-data` switch exists for that task.
+Consider following example.
+
+```bash
+echo -n "foo" > foo
+mkdir bar
+echo -n "baz" > bar/baz
+python-appimage [your regular parameters] -x foo bar/*
+```
+
+User data included in such a way becomes accessible to the Python code
+contained within the AppImage in a form of regular files under the directory
+pointed to by `APPDIR` environment variable. Example of Python 3 script
+that reads these exemplary files is presented below.
+
+```python
+import os, pathlib
+for fileName in ("foo", "baz"):
+  print((pathlib.Path(os.getenv("APPDIR")) / fileName).read_text())
+```
+
+Above code, when executed, would print following output.
+
+```bash
+foo
+baz
+```
 
 ## Advanced packaging
 

python_appimage/__main__.py

@@ -7,6 +7,11 @@ import sys
 __all__ = ['main']
 
 
+def exists(path):
+    if not os.path.exists(path):
+        raise argparse.ArgumentTypeError("could not find: {}".format(path))
+    return os.path.abspath(path)
+
 def main():
     '''Entry point for the CLI
     '''
@@ -22,6 +27,8 @@ def main():
                                        help='Command to execute',
                                        dest='command')
 
+    parser.add_argument('-a', '--appimagetool-version',
+        help='set appimagetool version')
     parser.add_argument('-q', '--quiet', help='disable logging',
         dest='verbosity', action='store_const', const='ERROR')
     parser.add_argument('-v', '--verbose', help='print extra information',
@@ -73,6 +80,8 @@ def main():
                                   help='force pip in-tree-build',
                                   action='store_true',
                                   default=False)
+    build_app_parser.add_argument('-x', '--extra-data', type=exists,
+        help='extra application data (bundled under $APPDIR/)', nargs='+')
 
     list_parser = subparsers.add_parser('list',
         description='List Python versions installed in a manylinux image')
@@ -91,6 +100,10 @@ def main():
         from .utils import log
         log.set_level(args.verbosity)
 
+    if args.appimagetool_version:
+        from .utils import deps
+        deps.APPIMAGETOOL_VERSION = args.appimagetool_version
+
     # check if no arguments are passed
     if args.command is None:
         parser.print_help()

python_appimage/appimage/build.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 
 from ..utils.compat import decode
-from ..utils.deps import APPIMAGETOOL, ensure_appimagetool
+from ..utils.deps import ensure_appimagetool
 from ..utils.docker import docker_run
 from ..utils.fs import copy_tree
 from ..utils.log import debug, log
@@ -22,10 +22,10 @@ def build_appimage(appdir=None, destination=None):
         appdir = 'AppDir'
 
     log('BUILD', appdir)
-    ensure_appimagetool()
+    appimagetool = ensure_appimagetool()
 
     arch = platform.machine()
-    cmd = ['ARCH=' + arch, APPIMAGETOOL, '--no-appstream', appdir]
+    cmd = ['ARCH=' + arch, appimagetool, '--no-appstream', appdir]
     if destination is not None:
         cmd.append(destination)
     cmd = ' '.join(cmd)
@@ -45,7 +45,8 @@ def build_appimage(appdir=None, destination=None):
         elif out:
             out = out.replace('%', '%%')[:-1]
             for line in out.split(os.linesep):
-                if line.startswith('WARNING'):
+                if line.startswith('WARNING') and \
+                    not line[9:].startswith('zsyncmake command is missing'):
                     log('WARNING', line[9:])
                 elif line.startswith('Error'):
                     raise RuntimeError(line)

python_appimage/appimage/relocate.py

@@ -94,6 +94,8 @@ def patch_binary(path, libdir, recursive=True):
     else:
         excluded = _excluded_libs
 
+    deps = ldd(path) # Fetch deps before patching RPATH.
+
     ensure_patchelf()
     rpath = '\'' + system((PATCHELF, '--print-rpath', path)) + '\''
     relpath = os.path.relpath(libdir, os.path.dirname(path))
@@ -102,7 +104,6 @@ def patch_binary(path, libdir, recursive=True):
     if rpath != expected:
         system((PATCHELF, '--set-rpath', expected, path))
 
-    deps = ldd(path)
     for dep in deps:
         name = os.path.basename(dep)
         if name in excluded:
@@ -171,11 +172,11 @@ def relocate_python(python=None, appdir=None):
 
     # Set some key variables & paths
     if python:
-        FULLVERSION = system((python, '-c',
+        FULLVERSION = system((python, '-c', '"import sys; print(sys.version)"'))
-            '"import sys; print(\'{:}.{:}.{:}\'.format(*sys.version_info[:3]))"'))
         FULLVERSION = FULLVERSION.strip()
     else:
-        FULLVERSION = '{:}.{:}.{:}'.format(*sys.version_info[:3])
+        FULLVERSION = sys.version
+    FULLVERSION = FULLVERSION.split(None, 1)[0]
     VERSION = '.'.join(FULLVERSION.split('.')[:2])
     PYTHON_X_Y = 'python' + VERSION
     PIP_X_Y = 'pip' + VERSION
@@ -202,9 +203,21 @@ def relocate_python(python=None, appdir=None):
     PYTHON_LIB = PYTHON_PREFIX + '/lib'
     PYTHON_PKG = PYTHON_LIB + '/' + PYTHON_X_Y
 
+    if not os.path.exists(HOST_PKG):
+        paths = glob.glob(HOST_PKG + '*')
+        if paths:
+            HOST_PKG = paths[0]
+            PYTHON_PKG = PYTHON_LIB + '/' + os.path.basename(HOST_PKG)
+        else:
+            raise ValueError('could not find {0:}'.format(HOST_PKG))
+
     if not os.path.exists(HOST_INC):
-        HOST_INC += 'm'
+        paths = glob.glob(HOST_INC + '*')
-        PYTHON_INC += 'm'
+        if paths:
+            HOST_INC = paths[0]
+            PYTHON_INC = PYTHON_INC + '/' + os.path.basename(HOST_INC)
+        else:
+            raise ValueError('could not find {0:}'.format(HOST_INC))
 
 
     # Copy the running Python's install

python_appimage/commands/build/app.py

@@ -26,14 +26,16 @@ def _unpack_args(args):
     '''Unpack command line arguments
     '''
     return args.appdir, args.name, args.python_version, args.linux_tag,        \
-           args.python_tag, args.base_image, args.in_tree_build
+           args.python_tag, args.base_image, args.in_tree_build,               \
+           args.extra_data
 
 
 _tag_pattern = re.compile('python([^-]+)[-]([^.]+)[.]AppImage')
 _linux_pattern = re.compile('manylinux([0-9]+)_' + platform.machine())
 
 def execute(appdir, name=None, python_version=None, linux_tag=None,
-            python_tag=None, base_image=None, in_tree_build=False):
+            python_tag=None, base_image=None, in_tree_build=False,
+            extra_data=None):
     '''Build a Python application using a base AppImage
     '''
 
@@ -253,6 +255,12 @@ def execute(appdir, name=None, python_version=None, linux_tag=None,
                 'WARNING: Running pip as'
             )
 
+            git_warnings = (
+                re.compile(r'\s+Running command git (clone|checkout) '),
+                re.compile(r"\s+Branch '.*' set up to track remote"),
+                re.compile(r"\s+Switched to a new branch '.*'"),
+            )
+
             isolation_flag = '-sE' if python_version[0] == '2' else '-I'
             system(('./AppDir/AppRun', isolation_flag, '-m', 'pip', 'install', '-U', in_tree_build,
                    '--no-warn-script-location', 'pip'), exclude=deprecation)
@@ -279,8 +287,17 @@ def execute(appdir, name=None, python_version=None, linux_tag=None,
                     log('BUNDLE', requirement)
                 system(('./AppDir/AppRun', isolation_flag, '-m', 'pip', 'install', '-U', in_tree_build,
                        '--no-warn-script-location', requirement),
-                       exclude=(deprecation, '  Running command git clone'))
+                       exclude=(deprecation + git_warnings))
 
+        # Bundle auxilliary application data
+        if extra_data is not None:
+            for path in extra_data:
+                basename = os.path.basename(path)
+                log('BUNDLE', basename)
+                if os.path.isdir(path):
+                    copy_tree(path, 'AppDir/' + basename)
+                else:
+                    copy_file(path, 'AppDir/')
 
         # Bundle the entry point
         entrypoint_path = glob.glob(appdir + '/entrypoint.*')

python_appimage/commands/which.py

@@ -16,6 +16,9 @@ def _unpack_args(args):
 def execute(binary):
     '''Print the location of a binary dependency
     '''
-    path = os.path.join(os.path.dirname(deps.PATCHELF), binary)
+    if binary == 'appimagetool':
+        path = deps.ensure_appimagetool(dry=True)
+    else:
+        path = os.path.join(os.path.dirname(deps.PATCHELF), binary)
     if os.path.exists(path):
         print(path)

python_appimage/utils/deps.py

@@ -19,29 +19,47 @@ _ARCH = platform.machine()
 PREFIX = os.path.abspath(os.path.dirname(__file__) + '/..')
 '''Package installation prefix'''
 
-APPIMAGETOOL = os.path.expanduser('~/.local/bin/appimagetool')
+APPIMAGETOOL_DIR = os.path.expanduser('~/.local/bin')
 '''Location of the appimagetool binary'''
 
+APPIMAGETOOL_VERSION = '12'
+'''Version of the appimagetool binary'''
+
 EXCLUDELIST = PREFIX + '/data/excludelist'
 '''AppImage exclusion list'''
 
 PATCHELF = os.path.expanduser('~/.local/bin/patchelf')
 '''Location of the PatchELF binary'''
 
-
+def ensure_appimagetool(dry=False):
-def ensure_appimagetool():
     '''Fetch appimagetool from the web if not available locally
     '''
-    if os.path.exists(APPIMAGETOOL):
-        return False
 
+    if APPIMAGETOOL_VERSION == '12':
+        appimagetool_name = 'appimagetool'
+    else:
+        appimagetool_name = 'appimagetool-' + APPIMAGETOOL_VERSION
+    appimagetool = os.path.join(APPIMAGETOOL_DIR, appimagetool_name)
+    appdir_name = '.'.join(('', appimagetool_name, 'appdir', _ARCH))
+    appdir = os.path.join(APPIMAGETOOL_DIR, appdir_name)
+    apprun = os.path.join(appdir, 'AppRun')
+
+    if dry or os.path.exists(apprun):
+        return apprun
     appimage = 'appimagetool-{0:}.AppImage'.format(_ARCH)
-    baseurl = 'https://github.com/AppImage/AppImageKit/releases/'              \
+
-              'download/12'
+    if APPIMAGETOOL_VERSION in map(str, range(1, 14)):
+        repository = 'AppImageKit'
+    else:
+        repository = 'appimagetool'
+    baseurl = os.path.join(
+        'https://github.com/AppImage',
+        repository,
+        'releases/download',
+        APPIMAGETOOL_VERSION
+    )
     log('INSTALL', 'appimagetool from %s', baseurl)
 
-    appdir_name = '.appimagetool.appdir'.format(_ARCH)
-    appdir = os.path.join(os.path.dirname(APPIMAGETOOL), appdir_name)
     if not os.path.exists(appdir):
         make_tree(os.path.dirname(appdir))
         with TemporaryDirectory() as tmpdir:
@@ -50,10 +68,7 @@ def ensure_appimagetool():
             system(('./' + appimage, '--appimage-extract'))
             copy_tree('squashfs-root', appdir)
 
-    if not os.path.exists(APPIMAGETOOL):
+    return apprun
-        os.symlink(appdir_name + '/AppRun', APPIMAGETOOL)
-
-    return True
 
 
 # Installers for dependencies

python_appimage/utils/fs.py

@@ -1,8 +1,30 @@
-from distutils.dir_util import mkpath as _mkpath, remove_tree as _remove_tree
-from distutils.file_util import copy_file as _copy_file
 import errno
 import os
 
+try:
+    from distutils.dir_util import mkpath as _mkpath
+    from distutils.dir_util import remove_tree as _remove_tree
+    from distutils.file_util import copy_file as _copy_file
+
+except ImportError:
+    import shutil
+
+    def _mkpath(path):
+        os.makedirs(path, exist_ok=True)
+
+    def _remove_tree(path):
+        shutil.rmtree(path)
+
+    def _copy_file(source, destination, update=0):
+        if os.path.exists(source) and (
+            not update
+            or (
+                (not os.path.exists(destination))
+                or (os.path.getmtime(source) > os.path.getmtime(destination))
+            )
+        ):
+            shutil.copy(source, destination)
+
 from .log import debug
 
 

python_appimage/utils/system.py

@@ -41,9 +41,16 @@ def system(args, exclude=None, stdin=None):
     if err:
         err = decode(err)
         stripped = [line for line in err.split(os.linesep) if line]
+
+        def matches_pattern(line, pattern):
+            if isinstance(pattern, re.Pattern):
+                return bool(pattern.match(line))
+            return line.startswith(pattern)
+
         for pattern in exclude:
             stripped = [line for line in stripped
-                        if not line.startswith(pattern)]
+                        if not matches_pattern(line, pattern)]
+
         if stripped:
             # Tolerate single line warning(s)
             for line in stripped:

python_appimage/utils/tmp.py

@@ -20,5 +20,6 @@ def TemporaryDirectory():
     try:
         yield tmpdir
     finally:
+        debug('REMOVE', tmpdir)
         os.chdir(pwd)
         remove_tree(tmpdir)

scripts/update-appimages.py

@@ -233,20 +233,30 @@ def update(args):
     for meta in new_assets:
         release = releases[meta.release_tag()].release
         appimage = meta.appimage_name()
-        new_asset = release.upload_asset(
+        if meta.asset and (meta.asset.name == appimage):
-            path = f'{APPIMAGES_DIR}/{appimage}',
-            name = appimage
-        )
-        if meta.asset:
             meta.asset.delete_asset()
             update_summary.append(
-                f'- update {meta.formated_tag()}/{meta.abi} '
+                f'- update {meta.formated_tag()}/{meta.abi} {meta.version}'
-                    f'{meta.previous_version()} -> {meta.version}'
+            )
+            new_asset = release.upload_asset(
+                path = f'{APPIMAGES_DIR}/{appimage}',
+                name = appimage
             )
         else:
-            update_summary.append(
+            new_asset = release.upload_asset(
-                f'- add {meta.formated_tag()}/{meta.abi} {meta.version}'
+                path = f'{APPIMAGES_DIR}/{appimage}',
+                name = appimage
             )
+            if meta.asset:
+                meta.asset.delete_asset()
+                update_summary.append(
+                    f'- update {meta.formated_tag()}/{meta.abi} '
+                        f'{meta.previous_version()} -> {meta.version}'
+                )
+            else:
+                update_summary.append(
+                    f'- add {meta.formated_tag()}/{meta.abi} {meta.version}'
+                )
 
         meta.asset = new_asset
         assets[meta.tag][meta.abi] = meta
@@ -300,6 +310,10 @@ if __name__ == '__main__':
         action = 'store_true',
         default = False
     )
+    parser.add_argument('-m', '--manylinux',
+        help = 'target specific manylinux tags',
+        nargs = "+"
+    )
     parser.add_argument("-s", "--sha",
         help = "reference commit SHA"
     )
@@ -308,5 +322,9 @@ if __name__ == '__main__':
     )
 
     args = parser.parse_args()
+
+    if args.manylinux:
+        MANYLINUSES = args.manylinux
+
     sys.argv = sys.argv[:1] # Empty args for fake call
     update(args)