From 6fc902894d2f43f75424251f183cc9596234086d Mon Sep 17 00:00:00 2001 From: karliss Date: Sun, 30 Aug 2020 23:23:41 +0300 Subject: [PATCH] CMake windows python packaging (#2406) * First version of cpack scripts * Fix python binding building using CMake on windows * Package python and pyside in the cpack package * Run windeployqt as part of packaging to include the Qt dlls and vcruntime installer --- .appveyor.yml | 2 +- .github/workflows/ccpp.yml | 34 +++++++++-- docs/source/building.rst | 7 +++ scripts/bundle_python.ps1 | 2 + scripts/fetch_deps.sh | 12 ++-- src/CMakeLists.txt | 27 +++++++-- src/bindings/src_list.py | 5 +- src/cmake/FindShiboken2.cmake | 90 ----------------------------- src/cmake/Packaging.cmake | 17 ++++++ src/cmake/WindowsBundlePython.cmake | 6 ++ src/cmake/WindowsBundleQt.cmake | 10 ++++ 11 files changed, 103 insertions(+), 109 deletions(-) delete mode 100644 src/cmake/FindShiboken2.cmake create mode 100644 src/cmake/Packaging.cmake create mode 100644 src/cmake/WindowsBundlePython.cmake create mode 100644 src/cmake/WindowsBundleQt.cmake diff --git a/.appveyor.yml b/.appveyor.yml index e008c1ff..0c8217a7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,5 +1,5 @@ version: '1.11.0-git-{build}' -image: 'Previous Visual Studio 2019' +image: 'Visual Studio 2019' clone_depth: 1 # Build configuration diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index 395618e5..eff0bde9 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -97,17 +97,39 @@ jobs: -DBREAKPAD_FRAMEWORK_DIR="$BREAKPAD_FRAMEWORK_DIR" \ ../src && \ make -j4; - - name: cmake windows + - name: windows dependencies if: contains(matrix.os, 'windows-latest') shell: bash run: | pip install ninja scripts/fetch_deps.sh - export PATH=$PWD/cutter-deps/qt/bin:$PATH + - name: windows cmake + if: contains(matrix.os, 'windows-latest') + shell: cmd + run: | + set CUTTER_DEPS=%CD%\cutter-deps + set PATH=%CD%\cutter-deps\qt\bin;%PATH% + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + cd mkdir build cd build - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCUTTER_USE_BUNDLED_RADARE2=ON \ + set PACKAGE_NAME=cutter-git-x64.Windows + cmake ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCUTTER_USE_BUNDLED_RADARE2=ON ^ + -DCUTTER_ENABLE_PYTHON=ON ^ + -DCUTTER_ENABLE_PYTHON_BINDINGS=ON ^ + -DCUTTER_PACKAGE_DEPENDENCIES=ON ^ + -DCUTTER_ENABLE_DEPENDENCY_DOWNLOADS=ON ^ + -DCMAKE_PREFIX_PATH="%CUTTER_DEPS%\pyside" ^ + -DCPACK_PACKAGE_FILE_NAME=%PACKAGE_NAME% ^ + -G Ninja ^ ../src - cmake --build . + cmake --build . --config Release + cmake --build . --config Release --target package + echo ::set-env name=PACKAGE_NAME::%PACKAGE_NAME%.zip + - uses: actions/upload-artifact@v2 + if: env.PACKAGE_NAME != null + with: + name: ${{ env.PACKAGE_NAME }} + path: build/${{ env.PACKAGE_NAME }} diff --git a/docs/source/building.rst b/docs/source/building.rst index a57e2bda..cb9d46ac 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -240,7 +240,14 @@ Note that there are some major building options available: * ``CUTTER_ENABLE_PYTHON_BINDINGS`` automatically generate Python Bindings with Shiboken2, required for Python plugins! * ``CUTTER_ENABLE_KSYNTAXHIGHLIGHTING`` use KSyntaxHighlighting for code highlighting. * ``CUTTER_ENABLE_GRAPHVIZ`` enable Graphviz for graph layouts. +* ``CUTTER_EXTRA_PLUGIN_DIRS`` List of addition plugin locations. Useful when preparing package for Linux distros that have strict package layout rules. + +Cutter binary release options, not needed for most users and might not work easily outside CI environment: + * ``CUTTER_ENABLE_CRASH_REPORTS`` is used to compile Cutter with crash handling system enabled (Breakpad). +* ``CUTTER_ENABLE_DEPENDENCY_DOWNLOADS`` Enable downloading of dependencies. Setting to OFF doesn't affect any downloads done by r2 build. This option is used for preparing Cutter binary release packges. Turned off by default. +* ``CUTTER_PACKAGE_DEPENDENCIES`` During install step include the third party dependencies. This option is used for preparing Cutter binary release packges. + These options can be enabled or disabled from the command line arguments passed to CMake. For example, to build Cutter with support for Python plugins, you can run this command: diff --git a/scripts/bundle_python.ps1 b/scripts/bundle_python.ps1 index 80b11d6e..b421ed98 100644 --- a/scripts/bundle_python.ps1 +++ b/scripts/bundle_python.ps1 @@ -6,6 +6,8 @@ $py_base = "python" + $py_version[0] + $py_version[2] $py_platform = If ($arch -eq "x64") {"amd64"} Else {"win32"} $py_url = "https://www.python.org/ftp/python/${py_version}/python-${py_version}-embed-${py_platform}.zip" +Remove-Item .\python_embed -Recurse -ErrorAction SilentlyContinue +$ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget ${py_url} -OutFile python_embed.zip; Expand-Archive .\python_embed.zip -DestinationPath .\python_embed New-Item -ItemType directory -Force -Path $dist\$py_base Copy-Item .\python_embed\${py_base}.zip -Destination $dist\$py_base diff --git a/scripts/fetch_deps.sh b/scripts/fetch_deps.sh index 895eee95..b22cd2de 100755 --- a/scripts/fetch_deps.sh +++ b/scripts/fetch_deps.sh @@ -4,16 +4,16 @@ cd $(dirname "${BASH_SOURCE[0]}")/.. mkdir -p cutter-deps && cd cutter-deps LINUX_FILE="cutter-deps-linux.tar.gz" -LINUX_MD5=a964be5ab3bc78815c69a1c2e60ccfef -LINUX_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-linux.tar.gz +LINUX_MD5=31fd19443a3405d6b6097cbbd4c11fba +LINUX_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-linux.tar.gz MACOS_FILE="cutter-deps-macos.tar.gz" -MACOS_MD5=7c60f5e226916b1889350d29b1c627f1 -MACOS_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-macos.tar.gz +MACOS_MD5=e75041c04fc806437723a399028402af +MACOS_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-macos.tar.gz WIN_FILE="cutter-deps-win.tar.gz" -WIN_MD5=36360faf85ef1754f1b3af004eeaa010 -WIN_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-win.tar.gz +WIN_MD5=7c755404140f2e9945bfc13d2e645bb1 +WIN_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-win.tar.gz if [ "$OS" == "Windows_NT" ]; then FILE="${WIN_FILE}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5a71d564..ed084533 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,14 +19,15 @@ option(CUTTER_ENABLE_CRASH_REPORTS "Enable crash report system. Unused if CUTTER option(CUTTER_APPIMAGE_BUILD "Enable Appimage specific changes. Doesn't cause building of Appimage itself." OFF) tri_option(CUTTER_ENABLE_KSYNTAXHIGHLIGHTING "Use KSyntaxHighlighting" AUTO) tri_option(CUTTER_ENABLE_GRAPHVIZ "Enable use of graphviz for graph layout" AUTO) -option (option SHIBOKEN_EXTRA_OPTIONS "Extra options for shiboken generator") +set(SHIBOKEN_EXTRA_OPTIONS "" CACHE STRING "Extra options for shiboken generator") set(CUTTER_EXTRA_PLUGIN_DIRS "" CACHE STRING "List of addition plugin locations") +option(CUTTER_ENABLE_DEPENDENCY_DOWNLOADS "Enable downloading of dependencies. Setting to OFF doesn't affect any downloads done by r2 build." OFF) +option(CUTTER_PACKAGE_DEPENDENCIES "During install step include the third party dependencies." OFF) if(NOT CUTTER_ENABLE_PYTHON) set(CUTTER_ENABLE_PYTHON_BINDINGS OFF) endif() - # Parse Cutter.pro to get filenames include(QMakeProParse) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/Cutter.pro" @@ -65,13 +66,13 @@ else() endif() if(CUTTER_ENABLE_PYTHON) + find_package(PythonInterp REQUIRED) find_package(PythonLibs ${CUTTER_PYTHON_MIN} REQUIRED) include_directories(${PYTHON_INCLUDE_DIRS}) add_definitions(-DCUTTER_ENABLE_PYTHON) if(CUTTER_ENABLE_PYTHON_BINDINGS) - find_package(PythonInterp REQUIRED) # 5.12.3 => 5.12 if("${Qt5_VERSION}" MATCHES "^([0-9]+\\.[0-9]+)\\.[0-9]+") set(Shiboken2_VERSION_REQUIRED "${CMAKE_MATCH_1}") @@ -146,8 +147,13 @@ if(CUTTER_ENABLE_PYTHON_BINDINGS) include_directories("${BINDINGS_BUILD_DIR}/CutterBindings") + set(SHIBOKEN_OPTIONS) + if (WIN32) + list(APPEND SHIBOKEN_OPTIONS --avoid-protected-hack) + endif() + add_custom_command(OUTPUT ${BINDINGS_SOURCE} - COMMAND Shiboken2::shiboken2 --project-file="${BINDINGS_BUILD_DIR}/bindings.txt" ${SHIBOKEN_EXTRA_OPTIONS} + COMMAND Shiboken2::shiboken2 --project-file="${BINDINGS_BUILD_DIR}/bindings.txt" ${SHIBOKEN_OPTIONS} ${SHIBOKEN_EXTRA_OPTIONS} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/bindings/bindings.xml" "${BINDINGS_BUILD_DIR}/bindings.txt" IMPLICIT_DEPENDS CXX "${CMAKE_CURRENT_SOURCE_DIR}/bindings/bindings.h" COMMENT "Generating Python bindings with shiboken2") @@ -210,6 +216,12 @@ endif() target_link_libraries(Cutter PUBLIC Qt5::Core Qt5::Widgets Qt5::Gui PRIVATE Qt5::Svg Qt5::Network) target_link_libraries(Cutter PUBLIC ${RADARE2_TARGET}) if(CUTTER_ENABLE_PYTHON) + if (WIN32) + # On windows some of the Python STABLE API functions are in seperate library + # which isn't added by CMake. + get_filename_component(_PYTHON_LIB_DIR ${PYTHON_LIBRARIES} DIRECTORY) + target_link_directories(Cutter PRIVATE ${_PYTHON_LIB_DIR}) + endif() target_link_libraries(Cutter PRIVATE ${PYTHON_LIBRARIES}) if(CUTTER_ENABLE_PYTHON_BINDINGS) target_link_libraries(Cutter PRIVATE Shiboken2::libshiboken PySide2::pyside2) @@ -233,9 +245,12 @@ if(CUTTER_ENABLE_PYTHON) list(APPEND BINDINGS_INCLUDE_DIRS ${Qt5Core_INCLUDE_DIRS} ${Qt5Widgets_INCLUDE_DIRS} ${Qt5Gui_INCLUDE_DIRS}) list(APPEND BINDINGS_INCLUDE_DIRS ${Radare2_INCLUDE_DIRS}) list(APPEND BINDINGS_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}") - string(REPLACE ";" ":" BINDINGS_INCLUDE_DIRS "${BINDINGS_INCLUDE_DIRS}") + if (NOT WIN32) + string(REPLACE ";" ":" BINDINGS_INCLUDE_DIRS "${BINDINGS_INCLUDE_DIRS}") + endif() qmake_configure_file("${BINDINGS_SRC_DIR}/bindings.txt.in" "${BINDINGS_BUILD_DIR}/bindings.txt") + add_compile_definitions(WIN32_LEAN_AND_MEAN) endif() endif() @@ -279,3 +294,5 @@ if(UNIX AND NOT APPLE) DESTINATION "share/applications" COMPONENT Devel) endif() + +include(Packaging) diff --git a/src/bindings/src_list.py b/src/bindings/src_list.py index 4f27809f..3175b39e 100755 --- a/src/bindings/src_list.py +++ b/src/bindings/src_list.py @@ -30,7 +30,10 @@ def get_cpp_files_gen(args, include_package=True): def cmd_cmake(args): - sys.stdout.write(";".join(get_cpp_files_gen(args))) + files = get_cpp_files_gen(args) + if sys.platform == "win32": + files = map(lambda x: x.replace("\\", "/"), files) + sys.stdout.write(";".join(files)) def cmd_qmake(args): diff --git a/src/cmake/FindShiboken2.cmake b/src/cmake/FindShiboken2.cmake deleted file mode 100644 index 0d39965c..00000000 --- a/src/cmake/FindShiboken2.cmake +++ /dev/null @@ -1,90 +0,0 @@ - -set(_module Shiboken2) - -find_package(${_module} ${${_module}_FIND_VERSION} CONFIG) -set(_executable_target ${_module}::shiboken2) -set(_lib_target ${_module}::libshiboken) - -if(NOT ${_module}_FOUND) - include(PythonInfo) - find_python_site_packages(PYTHON_SITE_PACKAGES) - get_python_extension_suffix(PYTHON_EXTENSION_SUFFIX) - - find_library(SHIBOKEN_LIBRARY - NAMES - "shiboken2${PYTHON_EXTENSION_SUFFIX}" - "shiboken2${PYTHON_EXTENSION_SUFFIX}.${${_module}_FIND_VERSION_MAJOR}.${${_module}_FIND_VERSION_MINOR}" - PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2") - - find_path(SHIBOKEN_INCLUDE_DIR - shiboken.h - PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2_generator/include") - - find_file(SHIBOKEN_BINARY - shiboken2 - PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2_generator") -endif() - -if(TARGET ${_executable_target}) - get_target_property(_is_imported ${_executable_target} IMPORTED) - if(_is_imported) - get_target_property(_imported_location ${_executable_target} IMPORTED_LOCATION) - if(NOT _imported_location) - message(STATUS "Target ${_executable_target} does not specify its IMPORTED_LOCATION! Trying to find it ourselves...") - find_file(SHIBOKEN_BINARY - shiboken2 - PATHS "${SHIBOKEN_SHARED_LIBRARY_DIR}/../bin" - NO_DEFAULT_PATH) - if(NOT SHIBOKEN_BINARY) - set(_message_type WARNING) - if(${_module}_FIND_REQUIRED) - set(_message_type FATAL_ERROR) - endif() - message(${_message_type} "Failed to manually find executable for ${_module}") - return() - endif() - message(STATUS "IMPORTED_LOCATION for ${_executable_target} found: ${SHIBOKEN_BINARY}") - set_target_properties(${_executable_target} PROPERTIES IMPORTED_LOCATION "${SHIBOKEN_BINARY}") - endif() - endif() - - get_target_property(_is_imported ${_lib_target} IMPORTED) - if(_is_imported) - get_target_property(_imported_location ${_lib_target} IMPORTED_LOCATION) - if(NOT _imported_location) - message(STATUS "Target ${_lib_target} does not specify its IMPORTED_LOCATION! Trying to find it ourselves...") - find_library(SHIBOKEN_LIBRARY - NAMES - "shiboken2${SHIBOKEN_PYTHON_EXTENSION_SUFFIX}" - "shiboken2${SHIBOKEN_PYTHON_EXTENSION_SUFFIX}.${${_module}_FIND_VERSION_MAJOR}.${${_module}_FIND_VERSION_MINOR}" - PATHS "${SHIBOKEN_SHARED_LIBRARY_DIR}") - if(NOT SHIBOKEN_LIBRARY) - set(_message_type WARNING) - if(${_module}_FIND_REQUIRED) - set(_message_type FATAL_ERROR) - endif() - message(${_message_type} "Failed to manually find library for ${_module}") - return() - endif() - message(STATUS "IMPORTED_LOCATION for ${_lib_target} found: ${SHIBOKEN_LIBRARY}") - set_target_properties(${_lib_target} PROPERTIES IMPORTED_LOCATION "${SHIBOKEN_LIBRARY}") - endif() - endif() -else() - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(${_module} - FOUND_VAR ${_module}_FOUND - REQUIRED_VARS SHIBOKEN_LIBRARY SHIBOKEN_INCLUDE_DIR SHIBOKEN_BINARY - VERSION_VAR ${_module}_VERSION) - - add_library(${_lib_target} INTERFACE IMPORTED) - set_target_properties(${_lib_target} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${SHIBOKEN_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${SHIBOKEN_LIBRARY}") - - add_executable(${_executable_target} IMPORTED) - set_target_properties(${_executable_target} PROPERTIES - IMPORTED_LOCATION "${SHIBOKEN_BINARY}") -endif() - -mark_as_advanced(SHIBOKEN_INCLUDE_DIR SHIBOKEN_LIBRARY SHIBOKEN_BINARY) \ No newline at end of file diff --git a/src/cmake/Packaging.cmake b/src/cmake/Packaging.cmake new file mode 100644 index 00000000..f55d293e --- /dev/null +++ b/src/cmake/Packaging.cmake @@ -0,0 +1,17 @@ +set(CUTTER_DEPS "${CMAKE_CURRENT_SOURCE_DIR}/../cutter-deps") +if(WIN32) + set(CPACK_GENERATOR "ZIP") + + if (CUTTER_PACKAGE_DEPENDENCIES) + if (CUTTER_ENABLE_PYTHON) + if (CUTTER_ENABLE_DEPENDENCY_DOWNLOADS) + set(CPACK_INSTALL_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/WindowsBundlePython.cmake) + endif() + find_package(PythonInterp REQUIRED) + install(DIRECTORY ${CUTTER_DEPS}/pyside/lib/site-packages DESTINATION "python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") + install(FILES ${CUTTER_DEPS}/pyside/bin/shiboken2.abi3.dll ${CUTTER_DEPS}/pyside/bin/pyside2.abi3.dll DESTINATION .) + endif() + install(SCRIPT cmake/WindowsBundleQt.cmake) + endif() +endif() +include(CPack) diff --git a/src/cmake/WindowsBundlePython.cmake b/src/cmake/WindowsBundlePython.cmake new file mode 100644 index 00000000..c84837fd --- /dev/null +++ b/src/cmake/WindowsBundlePython.cmake @@ -0,0 +1,6 @@ +execute_process(COMMAND powershell "${CMAKE_CURRENT_LIST_DIR}/../../scripts/bundle_python.ps1" x64 "${CMAKE_INSTALL_PREFIX}" + WORKING_DIRECTORY ${CPACK_PACKAGE_DIRECTORY} + RESULT_VARIABLE SCRIPT_RESULT) +if (SCRIPT_RESULT) + message(FATAL_ERROR "Failed to bundle python") +endif() diff --git a/src/cmake/WindowsBundleQt.cmake b/src/cmake/WindowsBundleQt.cmake new file mode 100644 index 00000000..9a1ce0ab --- /dev/null +++ b/src/cmake/WindowsBundleQt.cmake @@ -0,0 +1,10 @@ +message("Running windeployqt") +execute_process(COMMAND windeployqt Cutter.exe + --plugindir "qtplugins" + --no-translations # Cutter currently isn't loading Qt translation file + WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX} + RESULT_VARIABLE SCRIPT_RESULT) +if (SCRIPT_RESULT) + message(FATAL_ERROR "Failed to bundle python") +endif() +file(WRITE "${CMAKE_INSTALL_PREFIX}/qt.conf" "[PATHS]\nPlugins = qtplugins")