From c985fdc1b1efab4c74242958748ed9af2f487e4c Mon Sep 17 00:00:00 2001 From: karliss Date: Sat, 19 Dec 2020 12:43:33 +0200 Subject: [PATCH] Run cutter appimage packaging in GHA. (#2490) * Run cutter appimage packaging in GHA. * Update cutter-deps. * Switch to older visual studio due to pyside having problems parsing MSVC headers. * Take GIL in the injected code fragment #2511. --- .appveyor.yml | 4 +- .github/workflows/ccpp.yml | 73 +++++++++++++++++++++++++------- cmake/BundledRizin.cmake | 2 +- dist/CMakeLists.txt | 16 +++++-- dist/appbundle_embed_python.sh | 4 +- scripts/appimage_embed_python.sh | 2 +- scripts/fetch_deps.sh | 12 +++--- scripts/prepare_python_macos.sh | 6 +-- src/CutterApplication.cpp | 2 +- src/bindings/bindings.xml | 1 + 10 files changed, 88 insertions(+), 34 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 47891e68..496c0547 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,5 +1,5 @@ version: '1.12.0-git-{build}' -image: 'Previous Visual Studio 2019' +image: 'Visual Studio 2017' clone_depth: 1 # Build configuration @@ -9,7 +9,7 @@ configuration: # Environment environment: NINJA_URL: https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip - VSVARSALLPATH: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat' + VSVARSALLPATH: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat' ARCH: x64 PYTHON: 'C:\Python36-x64' PACKAGE_NAME: cutter-git-x64.Windows diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index e5a8e3d5..63c6459c 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -13,17 +13,22 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04] #, ubuntu-16.04, macos-latest, windows-latest] + os: [ubuntu-18.04, ubuntu-16.04, macos-latest, windows-2016] python-version: [3.7.x] system-deps: [false] cc-override: [default] cxx-override: [default] include: - - os: ubuntu-16.04 + - os: windows-2016 + package: true + - os: ubuntu-16.04 # ensure that Cutter can be built at least in basic config on Ubuntu 16.04 using sytem libraries python-version: 3.5.10 system-deps: true cc-override: '/usr/bin/gcc-5' cxx-override: '/usr/bin/g++-5' + - os: ubuntu-16.04 # release package build + system-deps: false + package: true # Prevent one job from pausing the rest fail-fast: false steps: @@ -34,10 +39,10 @@ jobs: - name: apt dependencies if: contains(matrix.os, 'ubuntu') run: | - sudo apt-get install libgraphviz-dev mesa-common-dev + sudo apt-get install libgraphviz-dev mesa-common-dev libxkbcommon-x11-dev if [[ "${{ matrix.os }}" = "ubuntu-16.04" ]] then - sudo apt-get install ninja-build/xenial-backports + sudo apt-get install ninja-build/xenial-backports libxcb1-dev libxkbcommon-dev libxcb-*-dev if [[ "${{ matrix.system-deps }}" = "true" ]] then sudo apt-get install cmake @@ -85,6 +90,7 @@ jobs: export PKG_CONFIG_PATH="$CUTTER_DEPS_PYTHON_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" export LD_LIBRARY_PATH="`llvm-config --libdir`:$LD_LIBRARY_PATH" fi + set -euo pipefail #TODO: move to top once cutter-deps doesn't fail if [[ "${{ matrix.cc-override }}" != "default" ]] then export CC="${{matrix.cc-override}}" @@ -100,14 +106,21 @@ jobs: then cmake \ -G Ninja \ - -DCUTTER_ENABLE_PYTHON=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DPYTHON_LIBRARY="$CUTTER_DEPS_PYTHON_PREFIX/lib/libpython3.6m.so.1.0" \ - -DPYTHON_INCLUDE_DIR="$CUTTER_DEPS_PYTHON_PREFIX/include/python3.6m" \ + -DCUTTER_ENABLE_PYTHON=ON \ + -DPYTHON_LIBRARY="$CUTTER_DEPS_PYTHON_PREFIX/lib/libpython3.9.so" \ + -DPYTHON_INCLUDE_DIR="$CUTTER_DEPS_PYTHON_PREFIX/include/python3.9" \ -DPYTHON_EXECUTABLE="$CUTTER_DEPS_PYTHON_PREFIX/bin/python3" \ - -DCUTTER_ENABLE_PYTHON_BINDINGS=OFF \ + -DCUTTER_ENABLE_PYTHON_BINDINGS=ON \ + -DCUTTER_ENABLE_GRAPHVIZ=OFF \ -DCUTTER_ENABLE_CRASH_REPORTS=ON \ -DCUTTER_USE_BUNDLED_RIZIN=ON \ + -DCUTTER_APPIMAGE_BUILD=ON \ + -DCUTTER_ENABLE_PACKAGING=ON \ + -DCUTTER_ENABLE_DEPENDENCY_DOWNLOADS=ON \ + -DCUTTER_PACKAGE_RZ_GHIDRA=ON \ + -DCMAKE_INSTALL_PREFIX=appdir/usr \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ .. else cmake \ @@ -117,6 +130,34 @@ jobs: .. fi ninja + if [[ "${{ matrix.package || false }}" = "true" ]] + then + export CUTTER_VERSION=$(python ../scripts/get_version.py) + export VERSION=$CUTTER_VERSION + ninja install + "../scripts/appimage_embed_python.sh" appdir + # "../scripts/rz-ghidra.sh" -DCMAKE_INSTALL_PREFIX="`pwd`/appdir/usr" -DBUILD_CUTTER_PLUGIN=ON -DCUTTER_SOURCE_DIR="`pwd`/.." + APP_PREFIX=`pwd`/appdir/usr + # export LD_LIBRARY_PATH=${APP_PREFIX}/lib:$Shiboken2_ROOT/lib + export PATH=$PATH:${APP_PREFIX}/bin + # "../scripts/r2dec.sh" --prefix=`pwd`/appdir/usr + wget -c "https://github.com/probonopd/linuxdeployqt/releases/download/6/linuxdeployqt-6-x86_64.AppImage" + chmod a+x linuxdeployqt*.AppImage + rm -fv "../cutter-deps/qt/plugins/imageformats/libqjp2.so" + # ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop -executable=./appdir/usr/bin/python3 -bundle-non-qt-libs -no-strip -exclude-libs=libnss3.so,libnssutil3.so,libqjp2.so -ignore-glob=usr/lib/python3.9/**/* -verbose=2 + # exclude librzghidra cutter plugin because cutter and rz plugin is loaded manuallly as they are plugins linuxdeployqt doesn't know this + ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop \ + -executable=./appdir/usr/bin/python3 \ + -appimage \ + -no-strip -exclude-libs=libnss3.so,libnssutil3.so,libqjp2.so \ + -ignore-glob=usr/lib/python3.9/**/* \ + -ignore-glob=usr/share/RizinOrg/Cutter/plugins/native/librz_ghidra_cutter.so \ + -verbose=2 + find ./appdir -executable -type f -exec ldd {} \; | grep " => /usr" | cut -d " " -f 2-3 | sort | uniq + export APPIMAGE_FILE="Cutter-v$CUTTER_VERSION-x64.Linux.AppImage" + mv Cutter-*-x86_64.AppImage "$APPIMAGE_FILE" + echo PACKAGE_NAME=$APPIMAGE_FILE >> $GITHUB_ENV + fi - name: cmake macos shell: bash if: contains(matrix.os, 'macos') @@ -130,31 +171,32 @@ jobs: cd build cmake \ -DCMAKE_BUILD_TYPE=Release \ - -DPYTHON_LIBRARY="$CUTTER_DEPS_PYTHON_PREFIX/lib/libpython3.6m.dylib" \ - -DPYTHON_INCLUDE_DIR="$CUTTER_DEPS_PYTHON_PREFIX/include/python3.6m" \ + -DPYTHON_LIBRARY="$CUTTER_DEPS_PYTHON_PREFIX/lib/libpython3.9.dylib" \ + -DPYTHON_INCLUDE_DIR="$CUTTER_DEPS_PYTHON_PREFIX/include/python3.9" \ -DPYTHON_EXECUTABLE="$CUTTER_DEPS_PYTHON_PREFIX/bin/python3" \ -DCUTTER_ENABLE_PYTHON=ON \ -DCUTTER_ENABLE_PYTHON_BINDINGS=ON \ -DCUTTER_ENABLE_CRASH_REPORTS=ON \ -DCUTTER_USE_BUNDLED_RIZIN=ON \ + -DCUTTER_ENABLE_PACKAGING=ON \ -DBREAKPAD_FRAMEWORK_DIR="$BREAKPAD_FRAMEWORK_DIR" \ .. && \ make -j4; - name: windows dependencies - if: contains(matrix.os, 'windows-latest') + if: contains(matrix.os, 'windows') shell: bash run: | pip install ninja scripts/fetch_deps.sh choco install winflexbison3 - name: windows cmake - if: contains(matrix.os, 'windows-latest') + if: contains(matrix.os, 'windows') shell: cmd run: | set ARCH=x64 set CUTTER_DEPS=%CD%\cutter-deps set PATH=%CD%\cutter-deps\qt\bin;%PATH% - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 call scripts\prepare_breakpad.bat cd mkdir build @@ -165,9 +207,10 @@ jobs: -DCUTTER_USE_BUNDLED_RIZIN=ON ^ -DCUTTER_ENABLE_PYTHON=ON ^ -DCUTTER_ENABLE_PYTHON_BINDINGS=ON ^ + -DCUTTER_ENABLE_PACKAGING=ON ^ -DCUTTER_PACKAGE_DEPENDENCIES=ON ^ - -DCUTTER_PACKAGE_R2GHIDRA=ON ^ - -DCUTTER_PACKAGE_R2DEC=ON ^ + -DCUTTER_PACKAGE_RZ_GHIDRA=ON ^ + -DCUTTER_PACKAGE_R2DEC=OFF ^ -DCUTTER_ENABLE_DEPENDENCY_DOWNLOADS=ON ^ -DCUTTER_ENABLE_CRASH_REPORTS=ON ^ -DCMAKE_PREFIX_PATH="%CUTTER_DEPS%\pyside" ^ diff --git a/cmake/BundledRizin.cmake b/cmake/BundledRizin.cmake index f0479db7..a9b144e6 100644 --- a/cmake/BundledRizin.cmake +++ b/cmake/BundledRizin.cmake @@ -71,5 +71,5 @@ elseif (WIN32) COMPONENT Devel PATTERN "*.pdb" EXCLUDE) else () - install(DIRECTORY "${RIZIN_INSTALL_DIR}/" DESTINATION ".") + install(DIRECTORY "${RIZIN_INSTALL_DIR}/" DESTINATION "." USE_SOURCE_PERMISSIONS) endif() diff --git a/dist/CMakeLists.txt b/dist/CMakeLists.txt index 1d88bcf4..3a6fce50 100644 --- a/dist/CMakeLists.txt +++ b/dist/CMakeLists.txt @@ -22,7 +22,7 @@ if(WIN32) endif() if (CUTTER_PACKAGE_R2DEC AND CUTTER_ENABLE_DEPENDENCY_DOWNLOADS) install(CODE " - set(ENV{R_ALT_SRC_DIR} 1) + set(ENV{RZ_ALT_SRC_DIR} 1) set(ENV{PATH} \"\${CMAKE_INSTALL_PREFIX};\$ENV{PATH}\") execute_process(COMMAND powershell \"${CMAKE_CURRENT_SOURCE_DIR}/bundle_r2dec.ps1\" \"\${CMAKE_INSTALL_PREFIX}\" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} @@ -91,6 +91,17 @@ if(CUTTER_PACKAGE_RZ_GHIDRA) INSTALL_COMMAND "" ) endif() + if (WIN32) + set (GHIDRA_OPTIONS " + \"-DRIZIN_INSTALL_PLUGDIR=\${CMAKE_INSTALL_PREFIX}/${RIZIN_INSTALL_PLUGDIR}\" + -DCUTTER_INSTALL_PLUGDIR=plugins/native") + elseif (APPLE) + set (GHIDRA_OPTIONS " + \"-DRIZIN_INSTALL_PLUGDIR=\${CMAKE_INSTALL_PREFIX}/${RIZIN_INSTALL_PLUGDIR}\" + -DCUTTER_INSTALL_PLUGDIR=plugins/native") + else() + set (GHIDRA_OPTIONS "") + endif() install(CODE " execute_process( WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rz-ghidra-prefix/src/rz-ghidra-build @@ -100,8 +111,7 @@ if(CUTTER_PACKAGE_RZ_GHIDRA) -DCMAKE_BUILD_TYPE=Release \"-DCMAKE_PREFIX_PATH=\${CMAKE_INSTALL_PREFIX};\${CMAKE_INSTALL_PREFIX}/include/librz;\${CMAKE_INSTALL_PREFIX}/include/librz/sdb;${RZ_GHIDRA_PREFIX_PATH}\" -DCMAKE_INSTALL_PREFIX=\${CMAKE_INSTALL_PREFIX} - \"-DRIZIN_INSTALL_PLUGDIR=\${CMAKE_INSTALL_PREFIX}/${RIZIN_INSTALL_PLUGDIR}\" - -DCUTTER_INSTALL_PLUGDIR=plugins/native + ${GHIDRA_OPTIONS} -DBUILD_CUTTER_PLUGIN=ON -DBUILD_SLEIGH_PLUGIN=OFF -G Ninja diff --git a/dist/appbundle_embed_python.sh b/dist/appbundle_embed_python.sh index 1267a7c2..c41159f2 100755 --- a/dist/appbundle_embed_python.sh +++ b/dist/appbundle_embed_python.sh @@ -5,7 +5,7 @@ if ! [[ $# -eq 3 ]]; then exit 1 fi -python_version=python3.6 +python_version=python3.9 py_framework=$1 appbundle=$2 @@ -22,7 +22,7 @@ install_name_tool -change `otool -L "$executable" | sed -n "s/^[[:blank:]]*\([^[ echo "Cleaning up embedded Python Framework" cd "$appbundle/Contents/Frameworks/Python.framework" || exit 1 find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf || exit 1 -rm -r Versions/Current/Resources/* Versions/Current/lib/python3.6/test Versions/Current/lib/python3.6/idlelib Versions/Current/lib/python3.6/curses Versions/Current/lib/python3.6/lib2to3 || exit 1 +rm -r Versions/Current/Resources/* Versions/Current/lib/$python_version/test Versions/Current/lib/$python_version/idlelib Versions/Current/lib/$python_version/curses Versions/Current/lib/$python_version/lib2to3 || exit 1 echo "Checking if PySide2 is available" diff --git a/scripts/appimage_embed_python.sh b/scripts/appimage_embed_python.sh index 6715ed40..a7a98f03 100755 --- a/scripts/appimage_embed_python.sh +++ b/scripts/appimage_embed_python.sh @@ -5,7 +5,7 @@ if ! [[ $# -eq 1 ]]; then exit 1 fi -python_version=python3.6 +python_version=python3.9 python_prefix=$(pkg-config --variable=prefix python3) appdir=$1 diff --git a/scripts/fetch_deps.sh b/scripts/fetch_deps.sh index 9ef8fe74..42484a0a 100755 --- a/scripts/fetch_deps.sh +++ b/scripts/fetch_deps.sh @@ -4,16 +4,16 @@ cd $(dirname "${BASH_SOURCE[0]}")/.. mkdir -p cutter-deps && cd cutter-deps LINUX_FILE="cutter-deps-linux.tar.gz" -LINUX_MD5=31fd19443a3405d6b6097cbbd4c11fba -LINUX_URL=https://github.com/rizinorg/cutter-deps/releases/download/v12/cutter-deps-linux.tar.gz +LINUX_MD5=6cd6dff0b5e33d5ff62c57b4400603f5 +LINUX_URL=https://github.com/rizinorg/cutter-deps/releases/download/v13/cutter-deps-linux.tar.gz MACOS_FILE="cutter-deps-macos.tar.gz" -MACOS_MD5=e75041c04fc806437723a399028402af -MACOS_URL=https://github.com/rizinorg/cutter-deps/releases/download/v12/cutter-deps-macos.tar.gz +MACOS_MD5=4a7fe127d99dc6087c290d828aa19f6b +MACOS_URL=https://github.com/rizinorg/cutter-deps/releases/download/v13/cutter-deps-macos.tar.gz WIN_FILE="cutter-deps-win.tar.gz" -WIN_MD5=7c755404140f2e9945bfc13d2e645bb1 -WIN_URL=https://github.com/rizinorg/cutter-deps/releases/download/v12/cutter-deps-win.tar.gz +WIN_MD5=d867df10fb0c2b029faf737453014da6 +WIN_URL=https://github.com/rizinorg/cutter-deps/releases/download/v13/cutter-deps-win.tar.gz if [ "$OS" == "Windows_NT" ]; then FILE="${WIN_FILE}" diff --git a/scripts/prepare_python_macos.sh b/scripts/prepare_python_macos.sh index 94bcc484..126621e8 100755 --- a/scripts/prepare_python_macos.sh +++ b/scripts/prepare_python_macos.sh @@ -4,12 +4,12 @@ SCRIPTPATH=$(realpath "$(dirname "${BASH_SOURCE[0]}")") mkdir python && cd python -wget "https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tar.xz" || exit 1 -tar -xf Python-3.6.4.tar.xz || exit 1 +wget "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tar.xz" || exit 1 +tar -xf Python-3.9.1.tar.xz || exit 1 export PYTHON_FRAMEWORK_DIR="`pwd`/framework" -cd Python-3.6.4 || exit 1 +cd Python-3.9.1 || exit 1 CPPFLAGS="-I$(brew --prefix openssl)/include" LDFLAGS="-L$(brew --prefix openssl)/lib" ./configure --enable-framework=$PYTHON_FRAMEWORK_DIR || exit 1 diff --git a/src/CutterApplication.cpp b/src/CutterApplication.cpp index b515d357..476aecd8 100644 --- a/src/CutterApplication.cpp +++ b/src/CutterApplication.cpp @@ -100,7 +100,7 @@ CutterApplication::CutterApplication(int &argc, char **argv) : QApplication(argc #ifdef Q_OS_WIN // Redefine rz_sys_prefix() behaviour - qputenv("R_ALT_SRC_DIR", "1"); + qputenv("RZ_ALT_SRC_DIR", "1"); #endif Core()->initialize(clOptions.enableRizinPlugins); diff --git a/src/bindings/bindings.xml b/src/bindings/bindings.xml index 1986d885..36386558 100644 --- a/src/bindings/bindings.xml +++ b/src/bindings/bindings.xml @@ -18,6 +18,7 @@