Improve macOS bundling (#1067)

* Improve macOS bundling
* Bundle libs for macOS CI
* Add MACOS_BUILD_DMG CMake option and make the CI upload the DMG so we don't lose executable permissions.
* Manually copy plugins if macdeployqt doesn't
* Ad-hoc codesign the app

5 files changed, 85 insertions, 23 deletions

diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
index 3bcc4df..e007e13 100644
--- a/.github/workflows/build-macos.yml
+++ b/.github/workflows/build-macos.yml
@@ -26,15 +26,12 @@ jobs:
     - name: Configure
       working-directory: ${{runner.workspace}}/build
       run: |
-          export PKG_CONFIG_PATH="$(brew --prefix libarchive)/lib/pkgconfig"
-          cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DQt5_DIR=$(brew --prefix qt5)/lib/cmake/Qt5
+          cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PREFIX_PATH="$(brew --prefix qt@5);$(brew --prefix libarchive)" -DMACOS_BUNDLE_LIBS=ON -DMACOS_BUILD_DMG=ON
     - name: Make
       working-directory: ${{runner.workspace}}/build
       run: |
         make -j$(sysctl -n hw.ncpu)
-        mkdir dist
-        cp -r melonDS.app dist
     - uses: actions/upload-artifact@v1
       with:
-        name: melonDS.app
-        path: ${{runner.workspace}}/build/dist
+        name: melonDS.dmg
+        path: ${{runner.workspace}}/build/melonDS.dmg
diff --git a/README.md b/README.md
index 63f25d4..d884f35 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,7 @@ As for the rest, the interface should be pretty straightforward. If you have a q
    ```bash
    cmake .. -G "MSYS Makefiles"
    make -j$(nproc --all)
-   ../msys-dist.sh
+   ../tools/msys-dist.sh
    ```
 If everything went well, melonDS and the libraries it needs should now be in the `dist` folder.
 
diff --git a/src/frontend/qt_sdl/CMakeLists.txt b/src/frontend/qt_sdl/CMakeLists.txt
index d47005c..b4da103 100644
--- a/src/frontend/qt_sdl/CMakeLists.txt
+++ b/src/frontend/qt_sdl/CMakeLists.txt
@@ -3,7 +3,7 @@ project(qt_sdl)
 SET(SOURCES_QT_SDL
     main.cpp
     main_shaders.h
-	CheatsDialog.cpp
+    CheatsDialog.cpp
     EmuSettingsDialog.cpp
     InputConfigDialog.cpp
     VideoSettingsDialog.cpp
@@ -18,7 +18,7 @@ SET(SOURCES_QT_SDL
     font.h
     Platform.cpp
     PlatformConfig.cpp
-	QPathInput.h
+    QPathInput.h
     
     ArchiveUtil.h
     ArchiveUtil.cpp
@@ -68,9 +68,7 @@ target_link_libraries(melonDS ${CMAKE_THREAD_LIBS_INIT})
 
 target_include_directories(melonDS PRIVATE ${SDL2_INCLUDE_DIRS} ${SLIRP_INCLUDE_DIRS} ${LIBARCHIVE_INCLUDE_DIRS})
 target_link_directories(melonDS PRIVATE ${SDL2_LIBRARY_DIRS} ${SLIRP_LIBRARY_DIRS})
-if (NOT APPLE)
-    target_link_directories(melonDS PRIVATE ${LIBARCHIVE_LIBRARY_DIRS})
-endif()
+target_link_directories(melonDS PRIVATE ${LIBARCHIVE_LIBRARY_DIRS})
 
 target_include_directories(melonDS PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
 target_include_directories(melonDS PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
@@ -78,13 +76,13 @@ target_include_directories(melonDS PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../..")
 target_link_libraries(melonDS core)
 
 if (BUILD_STATIC)
-	target_link_libraries(melonDS -static ${SDL2_STATIC_LIBRARIES} ${SLIRP_STATIC_LIBRARIES} ${LIBARCHIVE_STATIC_LIBRARIES})
+    target_link_libraries(melonDS -static ${SDL2_STATIC_LIBRARIES} ${SLIRP_STATIC_LIBRARIES} ${LIBARCHIVE_STATIC_LIBRARIES})
 else()
-	target_link_libraries(melonDS ${SDL2_LIBRARIES} ${SLIRP_LIBRARIES} ${LIBARCHIVE_LIBRARIES})
+    target_link_libraries(melonDS ${SDL2_LIBRARIES} ${SLIRP_LIBRARIES} ${LIBARCHIVE_LIBRARIES})
 endif()
 
 if (NOT Iconv_IS_BUILT_IN)
-	target_link_libraries(melonDS ${Iconv_LIBRARIES})
+    target_link_libraries(melonDS ${Iconv_LIBRARIES})
 endif()
 
 if (UNIX)
@@ -122,16 +120,15 @@ if (APPLE)
     set_source_files_properties("${CMAKE_SOURCE_DIR}/melon.icns" PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
 
     option(MACOS_BUNDLE_LIBS "Bundle libraries with the app on macOS" OFF)
+    option(MACOS_BUILD_DMG "Build DMG image of the macOS application bundle" OFF)
 
     if (MACOS_BUNDLE_LIBS)
-        # Copy Qt plugins into the bundle
-        get_target_property(qtcocoa_location Qt5::QCocoaIntegrationPlugin LOCATION)
-        target_sources(melonDS PRIVATE "${qtcocoa_location}")
-        set_source_files_properties("${qtcocoa_location}" PROPERTIES MACOSX_PACKAGE_LOCATION MacOS/platforms)
-
-        get_target_property(qtmacstyle_location Qt5::QMacStylePlugin LOCATION)
-        target_sources(melonDS PRIVATE "${qtmacstyle_location}")
-        set_source_files_properties("${qtmacstyle_location}" PROPERTIES MACOSX_PACKAGE_LOCATION MacOS/styles)
+        if (MACOS_BUILD_DMG)
+            set(DMGARG "--dmg")
+        endif()
+        add_custom_command(TARGET melonDS POST_BUILD
+            COMMAND ${CMAKE_SOURCE_DIR}/tools/mac-libs.sh ${DMGARG} ${CMAKE_BINARY_DIR}
+            COMMENT "Bundling macOS libraries...")
     endif()
 endif()
 
diff --git a/tools/mac-libs.sh b/tools/mac-libs.sh
new file mode 100755
index 0000000..1766cf7
--- /dev/null
+++ b/tools/mac-libs.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+
+set -o errexit
+set -o pipefail
+
+build_dmg=0
+app=melonDS.app
+
+if [[ "$1" == "--dmg" ]]; then
+	build_dmg=1
+	shift
+fi
+
+if [[ ! -d "$1" ]]; then
+	echo "Usage: $0 [--dmg] <build-dir>"
+	exit 1
+fi
+
+cd "$1"
+
+# macOS does not have the -f flag for readlink
+abspath() {
+	perl -MCwd -le 'print Cwd::abs_path shift' "$1"
+}
+
+cmake_qtdir=$(grep -E "Qt._DIR" CMakeCache.txt | cut -d= -f2)
+qtdir="$(abspath "$cmake_qtdir"/../../..)"
+
+if [[ ! -d "$app/Contents/Frameworks" ]]; then
+	"${qtdir}/bin/macdeployqt" "$app"
+fi
+
+# We'll have to copy the Qt plugins we need on our own if macdeployqt forgets
+# Qt6 bug?
+plugindir="$app/Contents/PlugIns"
+if [[ ! -d "$plugindir" ]]; then
+    mkdir -p "$plugindir/styles" "$plugindir/platforms"
+    cp "$qtdir/share/qt/plugins/styles/libqmacstyle.dylib" "$plugindir/styles/"
+    cp "$qtdir/share/qt/plugins/platforms/libqcocoa.dylib" "$plugindir/platforms/"
+
+    install_name_tool -add_rpath "@executable_path/../Frameworks" "$app/Contents/MacOS/melonDS"
+fi
+
+# Fix library load paths that macdeployqt forgot about
+fixup_libs() {
+	local libs=($(otool -L "$1" | sed -E 's/\t(.*) \(.*$/\1/' | grep -vE '/System|/usr/lib|\.framework|^\@|:$'))
+	
+	for lib in "${libs[@]}"; do
+		local base="$(basename "$lib")"
+		install_name_tool -change "$lib" "@executable_path/../Frameworks/$base" "$1"
+	done
+}
+
+find "$app/Contents/Frameworks" -maxdepth 1 -name '*.dylib' | while read lib; do
+	fixup_libs "$lib"
+done
+
+fixup_libs "$app/Contents/MacOS/melonDS"
+codesign -s - --deep "$app"
+
+if [[ $build_dmg == 1 ]]; then
+	mkdir dmg
+	cp -r "$app" dmg/
+	ln -s /Applications dmg/Applications
+	hdiutil create -volname melonDS -srcfolder dmg -ov -format UDZO melonDS.dmg
+	rm -r dmg
+fi
diff --git a/msys-dist.sh b/tools/msys-dist.sh
index d95a6d3..d95a6d3 100755
--- a/msys-dist.sh
+++ b/tools/msys-dist.sh