Use PyInstaller to freeze and then manually create AppImage

Signed-off-by: Dan Yeaw <dan@yeaw.me>

.bettercodehub.yml

@@ -5,6 +5,6 @@ exclude:
 - /gaphor/UML/uml2\.py
 - /docs/.*
 - /utils/.*
-- /win-installer/.*
+- /windows/.*
 - .*\.gaphor
 - .*\.svg

.github/workflows/build.yml

@@ -200,11 +200,11 @@ jobs:
       - name: Create macOS Application
         run: |
           make dist
-          cd packaging 
+          cd packaging
           make all
       - name: Import codesign certificate
         uses: apple-actions/import-codesign-certs@v1
-        with: 
+        with:
           p12-file-base64: ${{ secrets.BASE64_ENCODED_P12 }}
           p12-password: ${{ secrets.CERTPASSWORD_P12 }}
       - name: Sign app

.gitignore

@@ -38,10 +38,10 @@ pip-wheel-metadata
 .idea
 
 # Windows Install
-win-installer/gaphor-script.py
-win-installer/_build_root
-win-installer/file_version_info.txt
-win-installer/output
+packaging/windows/gaphor-script.py
+packaging/windows/_build_root
+packaging/windows/file_version_info.txt
+packaging/windows/output
 
 # MacOS
 package/gaphor.iconset/

appimage/entrypoint.sh.in

@@ -1,2 +0,0 @@
-#! /bin/bash -i
-{{ python-executable }} -u "${APPDIR}/opt/python{{ python-version }}/bin/gaphor" "$@"

docs/windows.md

@@ -35,6 +35,6 @@ script that creates a Windows installer using
 1. Follow the instructions for settings up a development environment above
 1. Run ``C:\msys64\mingw64.exe`` - a terminal window should pop up
 ```bash
-$ cd win-installer
+$ cd packaging/windows
 $ ./build-installer.sh
 ```

packaging/appimage/Makefile

@@ -8,11 +8,12 @@ ID := Gaphor-x86_64
 # Do not change version by hand!
 VERSION := 2.1.1
 
+BUILD := build
 DIST := dist
 
 all: dist
 
-dist: requirements.txt entrypoint.sh $(ID).AppImage
+dist: requirements.txt entrypoint.sh $(DIST)/$(ID).AppImage
 
 dep:
 	pip install python_appimage
@@ -34,13 +35,19 @@ requirements.txt:
 	sed -i -r 's/(^.+);.*/\1/g' requirements.txt
 	echo 'gaphor==${VERSION}' >> requirements.txt
 
-$(ID).AppImage: requirements.txt entrypoint.sh
-	python-appimage build app $(CURDIR)
+$(DIST)/$(ID).AppImage: requirements.txt entrypoint.sh
+	./build-appimage.sh
+
+lint: $(DIST)/$(ID).AppImage
+	./appdir-lint.sh $(DIST)/AppRun
 
 clean:
 	rm -f entrypoint.sh
 	rm -f requirements.txt
-	rm -f $(ID).AppImage
+	rm -rf $(BUILD)
+	rm -rf $(DIST)
+	rm -rf pyinstvenv
+	rm -f $(subst $(notdir $(CURDIR)),,$(CURDIR))gaphor-script.py
 
 # for local testing:
 

packaging/appimage/build-appimage.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "${DIR}"
+
+APP_DIR="${DIR}"/dist/AppRun
+APP_ID="org.gaphor.Gaphor"
+VERSION="$(poetry version --no-ansi | cut -d' ' -f2)"
+
+python -m venv pyinstvenv
+
+pyinstvenv/bin/pip install ../../dist/gaphor-"${VERSION}"-py3-none-any.whl
+pyinstvenv/bin/pip install pyinstaller==4.1.0
+
+function build_pyinstaller {
+    python ../make-script.py ../../pyproject.toml > ../gaphor-script.py
+    pyinstvenv/bin/pyinstaller -y ../gaphor.spec
+}
+
+function remove_excluded_files {
+    # Remove excludelist files known as having bad side effects
+    echo "Removing excluded files"
+    while IFS= read -r line; do
+        file="$(echo "${line}" | cut -d' ' -f1)"
+        if [ ! "${file}" = "" ] && [ ! "${file}" = "#" ]; then
+            [ -f "dist/gaphor/${file}" ] && rm -fv "dist/gaphor/${file}"
+        fi
+    done < excludelist
+}
+
+function create_package {
+    # Inspired by https://github.com/nuxeo/nuxeo-drive
+    # Create the final AppImage
+    local app_name="Gaphor"
+    local output="dist/${app_name}-x86_64.AppImage"
+
+    echo "Adjusting file names to fit in the AppImage"
+    # PyInstaller + AppImage inspired by https://gitlab.com/scottywz/ezpyi/
+    [ -d "${APP_DIR}" ] && rm -rf "${APP_DIR}"
+    mv -v "dist/gaphor" "${APP_DIR}"
+    mv -v "${APP_DIR}/gaphor" "${APP_DIR}/AppRun"
+    ln -srv "${APP_DIR}/AppRun" "${APP_DIR}/gaphor"
+
+    echo "Copying icon"
+    cp -v "${APP_ID}.png" "${APP_DIR}/${APP_ID}.png"
+
+    echo "Copying metadata files"
+    mkdir -pv "${APP_DIR}/usr/share/metainfo"
+    cp -v "${APP_ID}".appdata.xml "${APP_DIR}"/usr/share/metainfo
+    mkdir -pv "${APP_DIR}"/usr/share/applications
+    cp -v "${APP_ID}.desktop" "${APP_DIR}/usr/share/applications/"
+    ln -srv "${APP_DIR}/usr/share/applications/${APP_ID}.desktop" "${APP_DIR}/${APP_ID}.desktop"
+
+    echo "Decompressing the AppImage tool"
+    mkdir -p build
+    cd build
+    [ -d "squashfs-root" ] && rm -frv "squashfs-root"
+    wget https://github.com/AppImage/AppImageKit/releases/latest/download/appimagetool-x86_64.AppImage
+    chmod +x appimagetool-x86_64.AppImage
+    ./appimagetool-x86_64.AppImage --appimage-extract
+    cd ..
+
+    echo "Creating the AppImage file"
+    # --no-appstream because appstreamcli is not easily installable on CentOS
+    ./build/squashfs-root/AppRun --no-appstream "${APP_DIR}" "${output}"
+
+    echo "Clean-up"
+    rm -rf squashfs-root
+}
+
+function main {
+    build_pyinstaller
+    remove_excluded_files
+    create_package
+}
+
+main "$@";

packaging/appimage/entrypoint.sh.in

@@ -0,0 +1,2 @@
+#! /bin/bash -i
+"${APPDIR}/gaphor-exe" "$@"

packaging/appimage/excludelist

@@ -0,0 +1,165 @@
+# This file lists libraries that we will assume to be present on the host system and hence
+# should NOT be bundled inside AppImages. This is a working document; expect it to change
+# over time. File format: one filename per line. Each entry should have a justification comment.
+
+# See the useful tool at https://abi-laboratory.pro/index.php?view=navigator&symbol=hb_buffer_set_cluster_level#result
+# to investigate issues with missing symbols.
+
+ld-linux.so.2
+ld-linux-x86-64.so.2
+libanl.so.1
+libBrokenLocale.so.1
+libcidn.so.1
+libcrypt.so.1
+libc.so.6
+libdl.so.2
+libm.so.6
+libmvec.so.1
+libnsl.so.1
+libnss_compat.so.2
+libnss_db.so.2
+libnss_dns.so.2
+libnss_files.so.2
+libnss_hesiod.so.2
+libnss_nisplus.so.2
+libnss_nis.so.2
+libpthread.so.0
+libresolv.so.2
+librt.so.1
+libthread_db.so.1
+libutil.so.1
+# These files are all part of the GNU C Library which should never be bundled.
+# List was generated from a fresh build of glibc 2.25.
+
+libstdc++.so.6
+# Workaround for:
+# usr/lib/libstdc++.so.6: version `GLIBCXX_3.4.21' not found
+
+libGL.so.1
+# Part of the video driver (OpenGL); present on any regular
+# desktop system, may also be provided by proprietary drivers.
+# Known to cause issues if it's bundled.
+
+libdrm.so.2
+# Workaround for:
+# Antergos Linux release 2015.11 (ISO-Rolling)
+# /usr/lib/libdrm_amdgpu.so.1: error: symbol lookup error: undefined symbol: drmGetNodeTypeFromFd (fatal)
+# libGL error: unable to load driver: swrast_dri.so
+# libGL error: failed to load driver: swrast
+# Unrecognized OpenGL version
+
+libxcb.so.1
+# Workaround for:
+# Fedora 23
+# symbol lookup error: /lib64/libxcb-dri3.so.0: undefined symbol: xcb_send_fd
+# Uncertain if this is required to be bundled for some distributions - if so we need to write a version check script and use LD_PRELOAD to load the system version if it is newer
+# Fedora 25:
+# undefined symbol: xcb_send_request_with_fds
+# https://github.com/AppImage/AppImages/issues/128
+
+libX11.so.6
+# Workaround for:
+# Fedora 23
+# symbol lookup error: ./lib/libX11.so.6: undefined symbol: xcb_wait_for_reply64
+# Uncertain if this is required to be bundled for some distributions - if so we need to write a version check script and use LD_PRELOAD to load the system version if it is newer
+
+libgio-2.0.so.0
+# Workaround for:
+# On Ubuntu, "symbol lookup error: /usr/lib/x86_64-linux-gnu/gtk-2.0/modules/liboverlay-scrollbar.so: undefined symbol: g_settings_new"
+
+# libgdk-x11-2.0.so.0 # Missing on openSUSE-Tumbleweed-KDE-Live-x86_64-Snapshot20170601-Media.iso
+# libgtk-x11-2.0.so.0 # Missing on openSUSE-Tumbleweed-KDE-Live-x86_64-Snapshot20170601-Media.iso
+
+libasound.so.2
+# Workaround for:
+# No sound, e.g., in VLC.AppImage (does not find sound cards)
+
+libgdk_pixbuf-2.0.so.0
+# Workaround for:
+# On Ubuntu, get (inkscape:25621): GdkPixbuf-WARNING **: Error loading XPM image loader: Image type 'xpm' is not supported
+
+libfontconfig.so.1
+# Workaround for:
+# Application stalls when loading fonts during application launch; e.g., KiCad on ubuntu-mate
+
+# Note, after discussion we do not exlude this, but we can use a dummy library that just does nothing
+# libselinux.so.1
+# Workaround for:
+# sed: error while loading shared libraries: libpcre.so.3: cannot open shared object file: No such file or directory
+# Some distributions, such as Arch Linux, do not come with libselinux.so.1 by default.
+# The solution is to bundle a dummy mock library:
+# echo "extern int is_selinux_enabled(void){return 0;}" >> selinux-mock.c
+# gcc -s -shared -o libselinux.so.1 -Wl,-soname,libselinux.so.1 selinux-mock.c
+# strip libselinux.so.1
+# More information: https://github.com/AppImage/AppImages/issues/83
+
+# The following are assumed to be part of the base system
+# Removing these has worked e.g., for Krita. Feel free to report if
+# you think that some of these should go into AppImages and why.
+libcom_err.so.2
+libcrypt.so.1
+libexpat.so.1
+libgcc_s.so.1
+libglib-2.0.so.0
+libgpg-error.so.0
+# libgssapi_krb5.so.2 # Disputed, seemingly needed by Arch Linux since Kerberos is named differently there
+# libgssapi.so.3 # Seemingly needed when running Ubuntu 14.04 binaries on Fedora 23
+# libhcrypto.so.4 # Missing on openSUSE LEAP 42.0
+# libheimbase.so.1 # Seemingly needed when running Ubuntu 14.04 binaries on Fedora 23
+# libheimntlm.so.0 # Seemingly needed when running Ubuntu 14.04 binaries on Fedora 23
+# libhx509.so.5 # Missing on openSUSE LEAP 42.0
+libICE.so.6
+# libidn.so.11 # Does not come with Solus by default
+# libk5crypto.so.3 # Runnning AppImage built on Debian 9 or Ubuntu 16.04 on an Archlinux fails otherwise; https://github.com/AppImage/AppImages/issues/301
+libkeyutils.so.1
+# libkrb5.so.26 # Disputed, seemingly needed by Arch Linux since Kerberos is named differently there. Missing on openSUSE LEAP 42.0
+# libkrb5.so.3 # Disputed, seemingly needed by Arch Linux since Kerberos is named differently there
+# libkrb5support.so.0 # Disputed, seemingly needed by Arch Linux since Kerberos is named differently there
+libp11-kit.so.0
+# libpcre.so.3 # Missing on Fedora 24, SLED 12 SP1, and openSUSE Leap 42.2
+# libroken.so.18 # Mission on openSUSE LEAP 42.0
+# libsasl2.so.2 # Seemingly needed when running Ubuntu 14.04 binaries on Fedora 23
+libSM.so.6
+libusb-1.0.so.0
+libuuid.so.1
+# libwind.so.0 # Missing on openSUSE LEAP 42.0
+libz.so.1
+
+# Potentially dangerous libraries
+libgobject-2.0.so.0
+
+# Workaround for:
+# Rectangles instead of fonts
+# https://github.com/AppImage/AppImages/issues/240
+libpangoft2-1.0.so.0
+libpangocairo-1.0.so.0
+libpango-1.0.so.0
+
+# Workaround for:
+# e.g., Spotify
+# relocation error: /lib/x86_64-linux-gnu/libgcrypt.so.20:
+# symbol gpgrt_lock_lock, version GPG_ERROR_1.0 not defined
+# in file libgpg-error.so.0 with link time reference
+libgpg-error.so.0
+
+libjack.so.0
+# it must match the ABI of the JACK server which is installed in the base system
+# rncbc confirmed this
+
+# Unsolved issue:
+# https://github.com/probonopd/linuxdeployqt/issues/35
+# Error initializing NSS with a persistent database (sql:/home/me/.pki/nssdb): libsoftokn3.so: cannot open shared object file: No such file or directory
+# Error initializing NSS without a persistent database: NSS error code: -5925
+# nss_error=-5925, os_error=0
+# libnss3.so should not be removed from the bundles, as this causes other issues, e.g.,
+# https://github.com/probonopd/linuxdeployqt/issues/35#issuecomment-256213517
+# and https://github.com/AppImage/AppImages/pull/114
+# libnss3.so
+
+# The following cannot be excluded, see
+# https://github.com/AppImage/AppImages/commit/6c7473d8cdaaa2572248dcc53d7f617a577ade6b
+# http://stackoverflow.com/questions/32644157/forcing-a-binary-to-use-a-specific-newer-version-of-a-shared-library-so
+# libssl.so.1
+# libssl.so.1.0.0
+# libcrypto.so.1
+# libcrypto.so.1.0.0

packaging/appimage/org.gaphor.Gaphor.desktop

@@ -3,9 +3,9 @@ Name=Gaphor
 Comment=The simple modeling tool
 Comment[nl]=De eenvoudige modelleertool
 Comment[pl]=Proste narzędzie do modelowania
-Exec=gaphor %f
+Exec=gaphor-exe %f
 Icon=org.gaphor.Gaphor
 Terminal=false
 Type=Application
 Categories=GTK;GNOME;Development;ProjectManagement;
-MimeType=application/x-gaphor
+MimeType=application/x-gaphor;

packaging/gaphor.spec

@@ -1,62 +0,0 @@
-from PyInstaller.utils.hooks import copy_metadata
-
-
-block_cipher = None
-
-a = Analysis(
-        ['gaphor-script.py'],
-        pathex=['../'],
-        binaries=[],
-        datas=[
-            ('../gaphor/diagram/*.glade', 'gaphor/diagram'),
-	    ('../gaphor/ui/layout.xml', 'gaphor/ui'),
-	    ('../gaphor/ui/layout.css', 'gaphor/ui'),
-	    ('../gaphor/ui/*.glade', 'gaphor/ui'),
-	    ('../gaphor/ui/*.png', 'gaphor/ui'),
-	    ('../gaphor/services/helpservice/*.svg', 'gaphor/services/helpservice'),
-	    ('../gaphor/services/helpservice/*.glade', 'gaphor/services/helpservice'),
-	    ('../gaphor/ui/icons/hicolor/scalable/actions/*.svg', 'gaphor/ui/icons/hicolor/scalable/actions'),
-	    ('../LICENSE.txt', 'gaphor'),
-	    ('../gaphor/locale/*', 'gaphor/locale')
-	]+copy_metadata('gaphor')+copy_metadata('gaphas'),
-        # Fixes issue with setuptools https://github.com/pypa/setuptools/issues/1963
-        hiddenimports=['pkg_resources.py2_warn'],
-        hookspath=['hooks'],
-        runtime_hooks=[],
-        excludes=['lib2to3', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
-        win_no_prefer_redirects=False,
-        win_private_assemblies=False,
-        cipher=block_cipher,
-        noarchive=False
-    )
-pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
-exe = EXE(
-          pyz,
-          a.scripts,
-          options=[],
-          exclude_binaries=True,
-          name='gaphor-exe',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          icon='windows/gaphor.ico',
-          version='windows/file_version_info.txt',
-          console=False
-      )
-coll = COLLECT(
-           exe,
-           a.binaries,
-           a.zipfiles,
-           a.datas,
-           strip=False,
-           upx=True,
-           name='gaphor'
-       )
-app = BUNDLE(
-          coll,
-          name='Gaphor.app',
-          icon='macos/gaphor.icns',
-          bundle_identifier='org.gaphor.gaphor',
-          version='ERSION',
-      )