CMake windows python packaging (#2406)

* First version of cpack scripts
* Fix python binding building using CMake on windows
* Package python and pyside in the cpack package
* Run windeployqt as part of packaging to include the Qt dlls and vcruntime installer

.appveyor.yml

@@ -1,5 +1,5 @@
 version: '1.11.0-git-{build}'
-image: 'Previous Visual Studio 2019'
+image: 'Visual Studio 2019'
 clone_depth: 1
 
 # Build configuration

.github/workflows/ccpp.yml

@@ -97,17 +97,39 @@ jobs:
                 -DBREAKPAD_FRAMEWORK_DIR="$BREAKPAD_FRAMEWORK_DIR" \
                 ../src && \
               make -j4;
-    - name: cmake windows
+    - name: windows dependencies
       if: contains(matrix.os, 'windows-latest')
       shell: bash
       run: |
         pip install ninja
         scripts/fetch_deps.sh
-        export PATH=$PWD/cutter-deps/qt/bin:$PATH
+    - name: windows cmake
+      if: contains(matrix.os, 'windows-latest')
+      shell: cmd
+      run: |
+        set CUTTER_DEPS=%CD%\cutter-deps
+        set PATH=%CD%\cutter-deps\qt\bin;%PATH%
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+        cd
         mkdir build
         cd build
-        cmake \
+        set PACKAGE_NAME=cutter-git-x64.Windows
-          -DCMAKE_BUILD_TYPE=Release \
+        cmake ^
-          -DCUTTER_USE_BUNDLED_RADARE2=ON \
+          -DCMAKE_BUILD_TYPE=Release ^
+          -DCUTTER_USE_BUNDLED_RADARE2=ON ^
+          -DCUTTER_ENABLE_PYTHON=ON ^
+          -DCUTTER_ENABLE_PYTHON_BINDINGS=ON ^
+          -DCUTTER_PACKAGE_DEPENDENCIES=ON ^
+          -DCUTTER_ENABLE_DEPENDENCY_DOWNLOADS=ON ^
+          -DCMAKE_PREFIX_PATH="%CUTTER_DEPS%\pyside" ^
+          -DCPACK_PACKAGE_FILE_NAME=%PACKAGE_NAME% ^
+          -G Ninja ^
           ../src
-        cmake --build .
+        cmake --build . --config Release
+        cmake --build . --config Release --target package
+        echo ::set-env name=PACKAGE_NAME::%PACKAGE_NAME%.zip
+    - uses: actions/upload-artifact@v2
+      if: env.PACKAGE_NAME != null
+      with:
+        name: ${{ env.PACKAGE_NAME }}
+        path: build/${{ env.PACKAGE_NAME }}

docs/source/building.rst

@@ -240,7 +240,14 @@ Note that there are some major building options available:
 * ``CUTTER_ENABLE_PYTHON_BINDINGS`` automatically generate Python Bindings with Shiboken2, required for Python plugins!
 * ``CUTTER_ENABLE_KSYNTAXHIGHLIGHTING`` use KSyntaxHighlighting for code highlighting.
 * ``CUTTER_ENABLE_GRAPHVIZ`` enable Graphviz for graph layouts.
+* ``CUTTER_EXTRA_PLUGIN_DIRS`` List of addition plugin locations. Useful when preparing package for Linux distros that have strict package layout rules.
+
+Cutter binary release options, not needed for most users and might not work easily outside CI environment: 
+
 * ``CUTTER_ENABLE_CRASH_REPORTS`` is used to compile Cutter with crash handling system enabled (Breakpad).
+* ``CUTTER_ENABLE_DEPENDENCY_DOWNLOADS`` Enable downloading of dependencies. Setting to OFF doesn't affect any downloads done by r2 build. This option is used for preparing Cutter binary release packges. Turned off by default.
+* ``CUTTER_PACKAGE_DEPENDENCIES`` During install step include the third party dependencies. This option is used for preparing Cutter binary release packges. 
+
 
 These options can be enabled or disabled from the command line arguments passed to CMake.
 For example, to build Cutter with support for Python plugins, you can run this command:

scripts/bundle_python.ps1

@@ -6,6 +6,8 @@ $py_base = "python" + $py_version[0] + $py_version[2]
 $py_platform = If ($arch -eq "x64") {"amd64"} Else {"win32"}
 $py_url = "https://www.python.org/ftp/python/${py_version}/python-${py_version}-embed-${py_platform}.zip"
 
+Remove-Item .\python_embed -Recurse -ErrorAction SilentlyContinue
+$ErrorActionPreference = 'Stop'
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget ${py_url} -OutFile python_embed.zip; Expand-Archive .\python_embed.zip -DestinationPath .\python_embed
 New-Item -ItemType directory -Force -Path $dist\$py_base
 Copy-Item .\python_embed\${py_base}.zip -Destination $dist\$py_base

scripts/fetch_deps.sh

@@ -4,16 +4,16 @@ cd $(dirname "${BASH_SOURCE[0]}")/..
 mkdir -p cutter-deps && cd cutter-deps
 
 LINUX_FILE="cutter-deps-linux.tar.gz"
-LINUX_MD5=a964be5ab3bc78815c69a1c2e60ccfef
+LINUX_MD5=31fd19443a3405d6b6097cbbd4c11fba
-LINUX_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-linux.tar.gz
+LINUX_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-linux.tar.gz
 
 MACOS_FILE="cutter-deps-macos.tar.gz"
-MACOS_MD5=7c60f5e226916b1889350d29b1c627f1
+MACOS_MD5=e75041c04fc806437723a399028402af
-MACOS_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-macos.tar.gz
+MACOS_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-macos.tar.gz
 
 WIN_FILE="cutter-deps-win.tar.gz"
-WIN_MD5=36360faf85ef1754f1b3af004eeaa010
+WIN_MD5=7c755404140f2e9945bfc13d2e645bb1
-WIN_URL=https://github.com/radareorg/cutter-deps/releases/download/v11/cutter-deps-win.tar.gz
+WIN_URL=https://github.com/radareorg/cutter-deps/releases/download/v12/cutter-deps-win.tar.gz
 
 if [ "$OS" == "Windows_NT" ]; then
 	FILE="${WIN_FILE}"

src/CMakeLists.txt

@@ -19,14 +19,15 @@ option(CUTTER_ENABLE_CRASH_REPORTS "Enable crash report system. Unused if CUTTER
 option(CUTTER_APPIMAGE_BUILD "Enable Appimage specific changes. Doesn't cause building of Appimage itself." OFF)
 tri_option(CUTTER_ENABLE_KSYNTAXHIGHLIGHTING "Use KSyntaxHighlighting" AUTO)
 tri_option(CUTTER_ENABLE_GRAPHVIZ "Enable use of graphviz for graph layout" AUTO)
-option (option SHIBOKEN_EXTRA_OPTIONS "Extra options for shiboken generator")
+set(SHIBOKEN_EXTRA_OPTIONS "" CACHE STRING "Extra options for shiboken generator")
 set(CUTTER_EXTRA_PLUGIN_DIRS "" CACHE STRING "List of addition plugin locations")
+option(CUTTER_ENABLE_DEPENDENCY_DOWNLOADS "Enable downloading of dependencies. Setting to OFF doesn't affect any downloads done by r2 build." OFF)
+option(CUTTER_PACKAGE_DEPENDENCIES "During install step include the third party dependencies." OFF)
 
 if(NOT CUTTER_ENABLE_PYTHON)
     set(CUTTER_ENABLE_PYTHON_BINDINGS OFF)
 endif()
 
-
 # Parse Cutter.pro to get filenames
 include(QMakeProParse)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/Cutter.pro"
@@ -65,13 +66,13 @@ else()
 endif()
 
 if(CUTTER_ENABLE_PYTHON)
+    find_package(PythonInterp REQUIRED)
     find_package(PythonLibs ${CUTTER_PYTHON_MIN} REQUIRED)
 
     include_directories(${PYTHON_INCLUDE_DIRS})
     add_definitions(-DCUTTER_ENABLE_PYTHON)
 
     if(CUTTER_ENABLE_PYTHON_BINDINGS)
-        find_package(PythonInterp REQUIRED)
         # 5.12.3 => 5.12
         if("${Qt5_VERSION}" MATCHES "^([0-9]+\\.[0-9]+)\\.[0-9]+")
             set(Shiboken2_VERSION_REQUIRED "${CMAKE_MATCH_1}")
@@ -146,8 +147,13 @@ if(CUTTER_ENABLE_PYTHON_BINDINGS)
 
     include_directories("${BINDINGS_BUILD_DIR}/CutterBindings")
 
+    set(SHIBOKEN_OPTIONS)
+    if (WIN32)
+        list(APPEND SHIBOKEN_OPTIONS --avoid-protected-hack)
+    endif()
+
     add_custom_command(OUTPUT ${BINDINGS_SOURCE}
-            COMMAND Shiboken2::shiboken2 --project-file="${BINDINGS_BUILD_DIR}/bindings.txt" ${SHIBOKEN_EXTRA_OPTIONS}
+            COMMAND Shiboken2::shiboken2 --project-file="${BINDINGS_BUILD_DIR}/bindings.txt" ${SHIBOKEN_OPTIONS} ${SHIBOKEN_EXTRA_OPTIONS}
             DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/bindings/bindings.xml" "${BINDINGS_BUILD_DIR}/bindings.txt"
             IMPLICIT_DEPENDS CXX "${CMAKE_CURRENT_SOURCE_DIR}/bindings/bindings.h"
             COMMENT "Generating Python bindings with shiboken2")
@@ -210,6 +216,12 @@ endif()
 target_link_libraries(Cutter PUBLIC Qt5::Core Qt5::Widgets Qt5::Gui PRIVATE  Qt5::Svg Qt5::Network)
 target_link_libraries(Cutter PUBLIC ${RADARE2_TARGET})
 if(CUTTER_ENABLE_PYTHON)
+    if (WIN32)
+        # On windows some of the Python STABLE API functions are in seperate library
+        # which isn't added by CMake.
+        get_filename_component(_PYTHON_LIB_DIR ${PYTHON_LIBRARIES} DIRECTORY)
+        target_link_directories(Cutter PRIVATE ${_PYTHON_LIB_DIR})
+    endif()
     target_link_libraries(Cutter PRIVATE ${PYTHON_LIBRARIES})
     if(CUTTER_ENABLE_PYTHON_BINDINGS)
         target_link_libraries(Cutter PRIVATE Shiboken2::libshiboken PySide2::pyside2)
@@ -233,9 +245,12 @@ if(CUTTER_ENABLE_PYTHON)
         list(APPEND BINDINGS_INCLUDE_DIRS ${Qt5Core_INCLUDE_DIRS} ${Qt5Widgets_INCLUDE_DIRS} ${Qt5Gui_INCLUDE_DIRS})
         list(APPEND BINDINGS_INCLUDE_DIRS ${Radare2_INCLUDE_DIRS})
         list(APPEND BINDINGS_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}")
-        string(REPLACE ";" ":" BINDINGS_INCLUDE_DIRS "${BINDINGS_INCLUDE_DIRS}")
+        if (NOT WIN32)
+            string(REPLACE ";" ":" BINDINGS_INCLUDE_DIRS "${BINDINGS_INCLUDE_DIRS}")
+        endif()
 
         qmake_configure_file("${BINDINGS_SRC_DIR}/bindings.txt.in" "${BINDINGS_BUILD_DIR}/bindings.txt")
+        add_compile_definitions(WIN32_LEAN_AND_MEAN)
     endif()
 endif()
 
@@ -279,3 +294,5 @@ if(UNIX AND NOT APPLE)
         DESTINATION "share/applications"
         COMPONENT Devel)
 endif()
+
+include(Packaging)

src/bindings/src_list.py

@@ -30,7 +30,10 @@ def get_cpp_files_gen(args, include_package=True):
 
 
 def cmd_cmake(args):
-    sys.stdout.write(";".join(get_cpp_files_gen(args)))
+    files = get_cpp_files_gen(args)
+    if sys.platform == "win32":
+        files = map(lambda x: x.replace("\\", "/"), files)
+    sys.stdout.write(";".join(files))
 
 
 def cmd_qmake(args):

src/cmake/FindShiboken2.cmake

@@ -1,90 +0,0 @@
-
-set(_module Shiboken2)
-
-find_package(${_module} ${${_module}_FIND_VERSION} CONFIG)
-set(_executable_target ${_module}::shiboken2)
-set(_lib_target ${_module}::libshiboken)
-
-if(NOT ${_module}_FOUND)
-    include(PythonInfo)
-    find_python_site_packages(PYTHON_SITE_PACKAGES)
-    get_python_extension_suffix(PYTHON_EXTENSION_SUFFIX)
-
-    find_library(SHIBOKEN_LIBRARY
-            NAMES
-            "shiboken2${PYTHON_EXTENSION_SUFFIX}"
-            "shiboken2${PYTHON_EXTENSION_SUFFIX}.${${_module}_FIND_VERSION_MAJOR}.${${_module}_FIND_VERSION_MINOR}"
-            PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2")
-
-    find_path(SHIBOKEN_INCLUDE_DIR
-            shiboken.h
-            PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2_generator/include")
-
-    find_file(SHIBOKEN_BINARY
-            shiboken2
-            PATH_SUFFIXES "${PYTHON_SITE_PACKAGES}/shiboken2_generator")
-endif()
-
-if(TARGET ${_executable_target})
-    get_target_property(_is_imported ${_executable_target} IMPORTED)
-    if(_is_imported)
-        get_target_property(_imported_location ${_executable_target} IMPORTED_LOCATION)
-        if(NOT _imported_location)
-            message(STATUS "Target ${_executable_target} does not specify its IMPORTED_LOCATION! Trying to find it ourselves...")
-            find_file(SHIBOKEN_BINARY
-                    shiboken2
-                    PATHS "${SHIBOKEN_SHARED_LIBRARY_DIR}/../bin"
-                    NO_DEFAULT_PATH)
-            if(NOT SHIBOKEN_BINARY)
-                set(_message_type WARNING)
-                if(${_module}_FIND_REQUIRED)
-                    set(_message_type FATAL_ERROR)
-                endif()
-                message(${_message_type} "Failed to manually find executable for ${_module}")
-                return()
-            endif()
-            message(STATUS "IMPORTED_LOCATION for ${_executable_target} found: ${SHIBOKEN_BINARY}")
-            set_target_properties(${_executable_target} PROPERTIES IMPORTED_LOCATION "${SHIBOKEN_BINARY}")
-        endif()
-    endif()
-
-    get_target_property(_is_imported ${_lib_target} IMPORTED)
-    if(_is_imported)
-        get_target_property(_imported_location ${_lib_target} IMPORTED_LOCATION)
-        if(NOT _imported_location)
-            message(STATUS "Target ${_lib_target} does not specify its IMPORTED_LOCATION! Trying to find it ourselves...")
-            find_library(SHIBOKEN_LIBRARY
-                    NAMES
-                    "shiboken2${SHIBOKEN_PYTHON_EXTENSION_SUFFIX}"
-                    "shiboken2${SHIBOKEN_PYTHON_EXTENSION_SUFFIX}.${${_module}_FIND_VERSION_MAJOR}.${${_module}_FIND_VERSION_MINOR}"
-                    PATHS "${SHIBOKEN_SHARED_LIBRARY_DIR}")
-            if(NOT SHIBOKEN_LIBRARY)
-                set(_message_type WARNING)
-                if(${_module}_FIND_REQUIRED)
-                    set(_message_type FATAL_ERROR)
-                endif()
-                message(${_message_type} "Failed to manually find library for ${_module}")
-                return()
-            endif()
-            message(STATUS "IMPORTED_LOCATION for ${_lib_target} found: ${SHIBOKEN_LIBRARY}")
-            set_target_properties(${_lib_target} PROPERTIES IMPORTED_LOCATION "${SHIBOKEN_LIBRARY}")
-        endif()
-    endif()
-else()
-    include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(${_module}
-            FOUND_VAR ${_module}_FOUND
-            REQUIRED_VARS SHIBOKEN_LIBRARY SHIBOKEN_INCLUDE_DIR SHIBOKEN_BINARY
-            VERSION_VAR ${_module}_VERSION)
-
-    add_library(${_lib_target} INTERFACE IMPORTED)
-    set_target_properties(${_lib_target} PROPERTIES
-            INTERFACE_INCLUDE_DIRECTORIES "${SHIBOKEN_INCLUDE_DIR}"
-            INTERFACE_LINK_LIBRARIES "${SHIBOKEN_LIBRARY}")
-
-    add_executable(${_executable_target} IMPORTED)
-    set_target_properties(${_executable_target} PROPERTIES
-            IMPORTED_LOCATION "${SHIBOKEN_BINARY}")
-endif()
-
-mark_as_advanced(SHIBOKEN_INCLUDE_DIR SHIBOKEN_LIBRARY SHIBOKEN_BINARY)

src/cmake/Packaging.cmake

@@ -0,0 +1,17 @@
+set(CUTTER_DEPS "${CMAKE_CURRENT_SOURCE_DIR}/../cutter-deps")
+if(WIN32)
+    set(CPACK_GENERATOR "ZIP")
+
+    if (CUTTER_PACKAGE_DEPENDENCIES)
+        if (CUTTER_ENABLE_PYTHON)
+            if (CUTTER_ENABLE_DEPENDENCY_DOWNLOADS)
+                set(CPACK_INSTALL_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/cmake/WindowsBundlePython.cmake)
+            endif()
+            find_package(PythonInterp REQUIRED)
+            install(DIRECTORY ${CUTTER_DEPS}/pyside/lib/site-packages DESTINATION "python${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}")
+            install(FILES ${CUTTER_DEPS}/pyside/bin/shiboken2.abi3.dll ${CUTTER_DEPS}/pyside/bin/pyside2.abi3.dll DESTINATION .)
+        endif()
+        install(SCRIPT cmake/WindowsBundleQt.cmake)
+    endif()
+endif()
+include(CPack)

src/cmake/WindowsBundlePython.cmake

@@ -0,0 +1,6 @@
+execute_process(COMMAND powershell "${CMAKE_CURRENT_LIST_DIR}/../../scripts/bundle_python.ps1" x64 "${CMAKE_INSTALL_PREFIX}"
+    WORKING_DIRECTORY ${CPACK_PACKAGE_DIRECTORY}
+    RESULT_VARIABLE SCRIPT_RESULT)
+if (SCRIPT_RESULT)
+    message(FATAL_ERROR "Failed to bundle python")
+endif()

src/cmake/WindowsBundleQt.cmake

@@ -0,0 +1,10 @@
+message("Running windeployqt")
+execute_process(COMMAND windeployqt Cutter.exe
+        --plugindir "qtplugins"
+        --no-translations # Cutter currently isn't loading Qt translation file
+    WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}
+    RESULT_VARIABLE SCRIPT_RESULT)
+if (SCRIPT_RESULT)
+    message(FATAL_ERROR "Failed to bundle python")
+endif()
+file(WRITE "${CMAKE_INSTALL_PREFIX}/qt.conf" "[PATHS]\nPlugins = qtplugins")