Use cutter-deps on Linux (#1217)

* Use cutter-deps for Linux

* Update CMakeLists.txt for newer PySide2

* Add llvm-config --libdir to LD_LIBRARY_PATH in Travis

* Update cutter-deps

* Fix Python Prefix for Deploy

* Update cutter-deps

* Install Jupyter in Travis

* Add update_deps.py and update

* Enable Python Bindings for qmake in Travis

* Use absolute path for src_list.py in qmake

* Use python directly for src_list.py in qmake

* Keep env for linuxdeployqt

* Embed PySide2 in AppImage

* Fix appimage_embed_python.sh

.gitignore

@@ -68,3 +68,6 @@ compile_commands.json
 
 # vscode
 **/.vscode
+
+# cutter-deps
+/cutter-deps

.travis.yml

@@ -23,16 +23,9 @@ matrix:
     env: BUILD_SYSTEM=cmake
 
 sudo: required
-dist: trusty
+dist: xenial
 
 addons:
-  apt:
-    sources:
-      - sourceline: "ppa:beineri/opt-qt596-trusty"
-    packages:
-      - qt59base
-      - qt59svg
-      - qt59webengine
   homebrew:
     update: true
     brewfile: scripts/Brewfile
@@ -41,9 +34,11 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then   export PATH="/usr/local/opt/qt/bin:$PATH" ; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then   source scripts/prepare_python_macos.sh    ; fi
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      source /opt/qt5*/bin/qt5*-env.sh
+      scripts/fetch_deps.sh && 
+      source cutter-deps/env.sh &&
+      export LD_LIBRARY_PATH="`llvm-config --libdir`:$LD_LIBRARY_PATH" &&
+      python3 -m pip install -r scripts/pip_requirements.txt
     ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then source scripts/prepare_python_linux.sh    ; fi
 
 before_script:
   - git submodule init ; git submodule update
@@ -60,9 +55,19 @@ script:
   - cd build
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
         if [[ "$BUILD_SYSTEM" == "qmake" ]]; then
-            PKG_CONFIG_PATH="$CUSTOM_PYTHON_PREFIX/lib/pkgconfig" qmake CUTTER_ENABLE_PYTHON=true CUTTER_ENABLE_PYTHON_BINDINGS=false CUTTER_ENABLE_JUPYTER=true CUTTER_ENABLE_QTWEBENGINE=false PREFIX=/usr APPIMAGE=1 ../src && make -j4;
+            qmake CUTTER_ENABLE_PYTHON=true CUTTER_ENABLE_PYTHON_BINDINGS=true CUTTER_ENABLE_JUPYTER=true CUTTER_ENABLE_QTWEBENGINE=false PREFIX=/usr APPIMAGE=1 ../src && make -j4;
         elif [[ "$BUILD_SYSTEM" == "cmake" ]]; then
-            cmake -DPYTHON_LIBRARY="$CUSTOM_PYTHON_PREFIX/lib/libpython3.6m.so.1.0" -DPYTHON_INCLUDE_DIR="$CUSTOM_PYTHON_PREFIX/include/python3.6m" -DPYTHON_EXECUTABLE="$CUSTOM_PYTHON_PREFIX/bin/python3" -DCUTTER_ENABLE_PYTHON=ON -DCUTTER_ENABLE_PYTHON_BINDINGS=OFF -DCUTTER_ENABLE_JUPYTER=ON -DCUTTER_ENABLE_QTWEBENGINE=OFF ../src && make -j4;
+            cmake
+                -DCMAKE_BUILD_TYPE=Release
+                -DPYTHON_LIBRARY="$CUTTER_DEPS_PYTHON_PREFIX/lib/libpython3.6m.so.1.0"
+                -DPYTHON_INCLUDE_DIR="$CUTTER_DEPS_PYTHON_PREFIX/include/python3.6m"
+                -DPYTHON_EXECUTABLE="$CUTTER_DEPS_PYTHON_PREFIX/bin/python3"
+                -DCUTTER_ENABLE_PYTHON=ON
+                -DCUTTER_ENABLE_PYTHON_BINDINGS=ON
+                -DCUTTER_ENABLE_JUPYTER=ON
+                -DCUTTER_ENABLE_QTWEBENGINE=OFF
+                ../src &&
+                make -j4;
        fi
     elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
         if [[ "$BUILD_SYSTEM" == "qmake" ]]; then
@@ -103,12 +108,11 @@ after_success:
       cp -r /usr/share/radare2 appdir/usr/share/ &&
       mkdir -p appdir/usr/bin/translations &&
       cp ../src/translations/*.qm appdir/usr/bin/translations/ &&
-      "$TRAVIS_BUILD_DIR/scripts/appimage_embed_python.sh" "$CUSTOM_PYTHON_PREFIX" appdir &&
+      "$TRAVIS_BUILD_DIR/scripts/appimage_embed_python.sh" appdir &&
       wget -c "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" &&
       chmod a+x linuxdeployqt*.AppImage &&
-      unset QTDIR; unset QT_PLUGIN_PATH ; unset LD_LIBRARY_PATH &&
-      LD_LIBRARY_PATH=$CUSTOM_PYTHON_PREFIX/lib ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs -no-strip -exclude-libs=libnss3.so,libnssutil3.so -ignore-glob=usr/lib/python3.6/** -verbose=2 &&
-      LD_LIBRARY_PATH=$CUSTOM_PYTHON_PREFIX/lib ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop -appimage -no-strip -exclude-libs=libnss3.so,libnssutil3.so -ignore-glob=usr/lib/python3.6/** -verbose=2 &&
+      ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs -no-strip -exclude-libs=libnss3.so,libnssutil3.so -ignore-glob=usr/lib/python3.6/** -verbose=2 &&
+      ./linuxdeployqt*.AppImage ./appdir/usr/share/applications/*.desktop -appimage -no-strip -exclude-libs=libnss3.so,libnssutil3.so -ignore-glob=usr/lib/python3.6/** -verbose=2 &&
       find ./appdir -executable -type f -exec ldd {} \; | grep " => /usr" | cut -d " " -f 2-3 | sort | uniq &&
       export APPIMAGE_FILE="Cutter-v$CUTTER_VERSION-x64.Linux.AppImage" &&
       mv Cutter-*-x86_64.AppImage "$APPIMAGE_FILE" &&

scripts/appimage_embed_python.sh

@@ -1,18 +1,37 @@
 #!/bin/bash
 
-if ! [[ $# -eq 2 ]]; then
-    echo "Usage: $0 [Python prefix] [appdir]"
+if ! [[ $# -eq 1 ]]; then
+    echo "Usage: $0 [appdir]"
     exit 1
 fi
 
-python_prefix=$1
-appdir=$2
+python_version=python3.6
+
+python_prefix=$(pkg-config --variable=prefix python3)
+appdir=$1
 
 echo "Embedding Python from prefix $python_prefix in appdir $appdir"
 
-cp -RT "$python_prefix" "$appdir/usr/" || exit 1
+mkdir -p "$appdir/usr"
 cd "$appdir/usr/" || exit 1
 
+cp -RT "$python_prefix" "." || exit 1
 echo "Cleaning up embedded Python"
 find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
-rm -r lib/python3.6/test lib/python3.6/idlelib lib/python3.6/curses lib/python3.6/lib2to3
+rm -r lib/$python_version/test lib/$python_version/idlelib lib/$python_version/curses lib/$python_version/lib2to3
+
+echo "Checking if PySide2 is available"
+
+pyside_prefix=$(pkg-config --variable=prefix pyside2)
+if [ $? -ne 0 ]; then
+	echo "PySide2 is not available, ignoring."
+	exit 0
+fi
+
+echo "PySide is at $pyside_prefix"
+
+if [ "$pyside_prefix" == "$python_prefix" ]; then
+	echo "Prefixes are equal, not copying anything from lib"
+else
+	cp -RT "$pyside_prefix/lib/$python_version" "lib/$python_version" || exit 1
+fi

scripts/fetch_deps.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cd $(dirname "${BASH_SOURCE[0]}")/..
+mkdir -p cutter-deps && cd cutter-deps
+
+FILE=cutter-deps.tar.gz
+MD5=0805fa6a1626ce787f952b300e2b321d
+URL=https://github.com/radareorg/cutter-deps/releases/download/v2/cutter-deps.tar.gz
+
+curl -L "$URL" -o "$FILE" || exit 1
+echo "$MD5 $FILE" | md5sum -c - || exit 1
+tar -xf "$FILE" || exit 1
+./relocate.sh || exit 1
+

scripts/prepare_python_linux.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-SCRIPTPATH=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
-
-mkdir -p python && cd python || exit 1
-
-export CUSTOM_PYTHON_PREFIX="`pwd`/prefix"
-
-wget "https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tar.xz" || exit 1
-tar -xf Python-3.6.4.tar.xz || exit 1
-
-cd Python-3.6.4 || exit 1
-echo "Building Python to install to prefix $CUSTOM_PYTHON_PREFIX"
-
-./configure --enable-shared --prefix=$CUSTOM_PYTHON_PREFIX || exit 1
-make -j4 || exit 1
-make install > /dev/null || exit 1
-
-cd ..
-
-echo "Patching libs in $CUSTOM_PYTHON_PREFIX/lib/python3.6/lib-dynload to have the correct rpath"
-wget https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2 || exit 1
-tar -xf patchelf-0.9.tar.bz2 || exit 1
-cd patchelf-0.9 || exit 1
-./configure || exit 1
-make || exit 1
-cd ..
-
-for lib in "$CUSTOM_PYTHON_PREFIX/lib/python3.6/lib-dynload"/*.so; do
-	echo "  patching $lib"
-    patchelf-0.9/src/patchelf --set-rpath '$ORIGIN/../..' "$lib" || exit 1
-done
-
-PYTHONHOME=$CUSTOM_PYTHON_PREFIX \
-       LD_LIBRARY_PATH=$CUSTOM_PYTHON_PREFIX/lib \
-       "$CUSTOM_PYTHON_PREFIX/bin/pip3" install -r "$SCRIPTPATH/pip_requirements.txt" || exit 1
-
-cd ..

scripts/update_deps.py

@@ -0,0 +1,32 @@
+
+import sys
+import os
+import requests
+import re
+import subprocess
+
+fetch_deps_path = os.path.join(os.path.dirname(sys.argv[0]), "fetch_deps.sh")
+
+print("Fetching latest release")
+json = requests.get("https://api.github.com/repos/radareorg/cutter-deps/releases/latest").json()
+
+release_url = json["assets"][0]["browser_download_url"]
+
+print(f"Getting MD5 for {release_url}")
+
+curl = subprocess.Popen(["curl", "-L", release_url], stdout=subprocess.PIPE)
+md5sum = subprocess.run(["md5sum"], stdin=curl.stdout, capture_output=True, encoding="utf-8").stdout
+curl.wait()
+
+md5sum = re.match("([a-zA-Z0-9]+) ", md5sum).group(1)
+
+print(f"MD5: {md5sum}")
+
+with open(fetch_deps_path) as f:
+	fetch_deps = f.read()
+
+fetch_deps = re.sub("^URL=.*$", f"URL={release_url}".replace("\\", r"\\"), fetch_deps, flags=re.MULTILINE)
+fetch_deps = re.sub("^MD5=.*$", f"MD5={md5sum}".replace("\\", r"\\"), fetch_deps, flags=re.MULTILINE)
+
+with open(fetch_deps_path, "w") as f:
+	f.write(fetch_deps)

src/CMakeLists.txt

@@ -1,6 +1,8 @@
 
 cmake_minimum_required(VERSION 3.1)
 
+cmake_policy(SET CMP0074 NEW)
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 include(DisallowInSource)
 
@@ -32,11 +34,11 @@ set(CUTTER_VERSION_PATCH "${CUTTER_PRO_CUTTER_VERSION_PATCH}")
 
 set(CUTTER_VERSION_FULL "${CUTTER_VERSION_MAJOR}.${CUTTER_VERSION_MINOR}.${CUTTER_VERSION_PATCH}")
 
-message(STATUS "version from Cutter.pro: ${CUTTER_VERSION_FULL}")
-message(STATUS "sources from Cutter.pro: ${SOURCE_FILES}")
-message(STATUS "headers from Cutter.pro: ${HEADER_FILES}")
-message(STATUS "forms from Cutter.pro: ${UI_FILES}")
-message(STATUS "resources from Cutter.pro: ${QRC_FILES}")
+#message(STATUS "version from Cutter.pro: ${CUTTER_VERSION_FULL}")
+#message(STATUS "sources from Cutter.pro: ${SOURCE_FILES}")
+#message(STATUS "headers from Cutter.pro: ${HEADER_FILES}")
+#message(STATUS "forms from Cutter.pro: ${UI_FILES}")
+#message(STATUS "resources from Cutter.pro: ${QRC_FILES}")
 
 project(Cutter VERSION "${CUTTER_VERSION_FULL}")
 
@@ -106,8 +108,27 @@ If you explicitly want to not generate bindings, re-run CMake with -DCUTTER_ENAB
 or, if you don't want to build with Python at all, use -DCUTTER_ENABLE_PYTHON=OFF.")
         endif()
 
+        if(NOT SHIBOKEN_BINARY)
+			# Newer Versions of Shiboken2 export targets
+            set(SHIBOKEN_BINARY Shiboken2::shiboken2)
+            set(SHIBOKEN_LIBRARY Shiboken2::libshiboken)
+        else()
 	        include_directories(${SHIBOKEN_INCLUDE_DIR})
-        include_directories(${PYSIDE_INCLUDE_DIR} ${PYSIDE_INCLUDE_DIR}/QtCore ${PYSIDE_INCLUDE_DIR}/QtGui ${PYSIDE_INCLUDE_DIR}/QtWidgets)
+		endif()
+
+		if(NOT PYSIDE_LIBRARY)
+			# Newer Versions of PySide2 export targets
+			set(PYSIDE_LIBRARY PySide2::pyside2)
+			get_target_property(PYSIDE_INCLUDE_DIR PySide2::pyside2 INTERFACE_INCLUDE_DIRECTORIES)
+			list(GET PYSIDE_INCLUDE_DIR 0 PYSIDE_INCLUDE_DIR)
+		endif()
+
+		if(PYSIDE_INCLUDE_DIR)
+		    include_directories(${PYSIDE_INCLUDE_DIR}
+				${PYSIDE_INCLUDE_DIR}/QtCore
+				${PYSIDE_INCLUDE_DIR}/QtGui
+				${PYSIDE_INCLUDE_DIR}/QtWidgets)
+		endif()
 
         add_definitions(-DCUTTER_ENABLE_PYTHON_BINDINGS)
     endif()

src/Cutter.pro

@@ -126,7 +126,6 @@ CUTTER_ENABLE_PYTHON {
         pythonpath = $$clean_path($$dirname(pythonpath))
         LIBS += -L$${pythonpath} -L$${pythonpath}/libs -lpython3
         INCLUDEPATH += $${pythonpath}/include
-        BINDINGS_SRC_LIST_CMD = "$${PYTHON_EXECUTABLE} bindings/src_list.py"
     }
 
     unix|macx|bsd {
@@ -142,7 +141,6 @@ CUTTER_ENABLE_PYTHON {
             }
             PKGCONFIG += python3
         }
-        BINDINGS_SRC_LIST_CMD = "bindings/src_list.py"
     }
 
     CUTTER_ENABLE_PYTHON_BINDINGS {
@@ -152,6 +150,11 @@ CUTTER_ENABLE_PYTHON {
         !packagesExist(pyside2) {
             error("ERROR: PySide2, which is required to build the Python Bindings, could not be found. Make sure it is available to pkg-config.")
         }
+        win32 {
+            BINDINGS_SRC_LIST_CMD = "$${PYTHON_EXECUTABLE} bindings/src_list.py"
+        } else {
+            BINDINGS_SRC_LIST_CMD = "python3 bindings/src_list.py"
+        }
         BINDINGS_SRC_DIR = "$${PWD}/bindings"
         BINDINGS_BUILD_DIR = "$${OUT_PWD}/bindings"
         BINDINGS_SOURCE = $$system("$${BINDINGS_SRC_LIST_CMD} qmake \"$${BINDINGS_BUILD_DIR}\"")
@@ -167,9 +170,9 @@ CUTTER_ENABLE_PYTHON {
         PYSIDE_TYPESYSTEMS = $$system("pkg-config --variable=typesystemdir pyside2")
         PYSIDE_INCLUDEDIR = $$system("pkg-config --variable=includedir pyside2")
         QMAKE_SUBSTITUTES += bindings/bindings.txt.in
-        #SHIBOKEN_EXECUTABLE = $$system("pkg-config --variable="
+        SHIBOKEN_EXECUTABLE = $$system("pkg-config --variable=generator_location shiboken2")
         bindings.target = bindings_target
-        bindings.commands = shiboken2 --project-file="$${BINDINGS_BUILD_DIR}/bindings.txt"
+        bindings.commands = "$${SHIBOKEN_EXECUTABLE}" --project-file="$${BINDINGS_BUILD_DIR}/bindings.txt"
         QMAKE_EXTRA_TARGETS += bindings
         GENERATED_SOURCES += $${BINDINGS_SOURCE}
         INCLUDEPATH += "$${BINDINGS_BUILD_DIR}/CutterBindings"