Graphviz based graph layout (#1691)

.travis.yml

@@ -19,7 +19,7 @@ matrix:
     os: linux
     env: BUILD_SYSTEM=cmake
     before_install:
-      - sudo apt-get update && sudo apt-get install ninja-build # because the apt addon is broken on bionic
+      - sudo apt-get update && sudo apt-get install ninja-build libgraphviz-dev # because the apt addon is broken on bionic
       - pyenv global 3.7.1
       - pip install meson
 
@@ -36,7 +36,7 @@ matrix:
   - name: Documentation + Deploy
     os: linux
     cache: ~
-    before_install: sudo apt-get update && sudo apt-get install doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme # because the apt addon is broken on bionic
+    before_install: sudo apt-get update && sudo apt-get install doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme libgraphviz-dev # because the apt addon is broken on bionic
     install: ~
     before_script: ~
     after_success: ~

src/CMakeLists.txt

@@ -16,6 +16,7 @@ option(CUTTER_ENABLE_PYTHON "Enable Python integration. Requires Python >= ${CUT
 option(CUTTER_ENABLE_PYTHON_BINDINGS "Enable generating Python bindings with Shiboken2. Unused if CUTTER_ENABLE_PYTHON=OFF." OFF)
 option(CUTTER_ENABLE_CRASH_REPORTS "Enable crash report system. Unused if CUTTER_ENABLE_CRASH_REPORTS=OFF" OFF)
 tri_option(CUTTER_ENABLE_KSYNTAXHIGHLIGHTING "Use KSyntaxHighlighting" AUTO)
+tri_option(CUTTER_ENABLE_GRAPHVIZ "Enable use of gprahviz for graph layout" AUTO)
 
 if(NOT CUTTER_ENABLE_PYTHON)
     set(CUTTER_ENABLE_PYTHON_BINDINGS OFF)
@@ -102,7 +103,20 @@ else()
     set(KSYNTAXHIGHLIGHTING_STATUS OFF)
 endif()
 
+find_package(PkgConfig REQUIRED)
 
+if (CUTTER_ENABLE_GRAPHVIZ)
+    if (CUTTER_ENABLE_GRAPHVIZ STREQUAL AUTO)
+        pkg_check_modules(GVC libgvc)
+        if (GVC_FOUND)
+            set(CUTTER_ENABLE_GRAPHVIZ ON)
+        else()
+            set(CUTTER_ENABLE_GRAPHVIZ OFF)
+        endif()
+    else()
+        pkg_check_modules(GVC REQUIRED libgvc)
+    endif()
+endif()
 
 message(STATUS "")
 message(STATUS "Building Cutter version ${CUTTER_VERSION_FULL}")
@@ -112,6 +126,7 @@ message(STATUS "- Python: ${CUTTER_ENABLE_PYTHON}")
 message(STATUS "- Python Bindings: ${CUTTER_ENABLE_PYTHON_BINDINGS}")
 message(STATUS "- Crash Handling: ${CUTTER_ENABLE_CRASH_REPORTS}")
 message(STATUS "- KSyntaxHighlighting: ${KSYNTAXHIGHLIGHTING_STATUS}")
+message(STATUS "- Graphviz: ${CUTTER_ENABLE_GRAPHVIZ}")
 message(STATUS "")
 
 
@@ -142,6 +157,11 @@ else()
 endif()
 
 
+if (CUTTER_ENABLE_GRAPHVIZ)
+    list(APPEND SOURCE_FILES ${CUTTER_PRO_GRAPHVIZ_SOURCES})
+    list(APPEND HEADER_FILES ${CUTTER_PRO_GRAPHVIZ_HEADERS})
+endif()
+
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
         OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -153,6 +173,13 @@ endif()
 add_executable(Cutter MACOSX_BUNDLE ${UI_FILES} ${QRC_FILES} ${SOURCE_FILES} ${HEADER_FILES} ${BINDINGS_SOURCE})
 set_target_properties(Cutter PROPERTIES MACOSX_BUNDLE_INFO_PLIST "${CMAKE_CURRENT_SOURCE_DIR}/macos/Info.plist")
 
+if (CUTTER_ENABLE_GRAPHVIZ)
+    target_link_libraries(Cutter ${GVC_LIBRARIES})
+    target_include_directories(Cutter PUBLIC ${GVC_INCLUDE_DIRS})
+    target_compile_options(Cutter PUBLIC ${GVC_CFLAGS_OTHER})
+    target_compile_definitions(Cutter PRIVATE CUTTER_ENABLE_GRAPHVIZ)
+endif()
+
 if(CUTTER_ENABLE_CRASH_REPORTS)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
     find_package(Threads REQUIRED)

src/Cutter.pro

@@ -362,6 +362,9 @@ SOURCES += \
     common/SelectionHighlight.cpp \
     common/Decompiler.cpp
 
+GRAPHVIZ_SOURCES = \
+    widgets/GraphvizLayout.cpp
+
 HEADERS  += \
     core/Cutter.h \
     core/CutterCommon.h \
@@ -485,11 +488,12 @@ HEADERS  += \
     common/BugReporting.h \
     common/HighDpiPixmap.h \
     widgets/GraphLayout.h \
-    widgets/GraphGridLayout.h \
     widgets/HexWidget.h \
     common/SelectionHighlight.h \
     common/Decompiler.h
 
+GRAPHVIZ_HEADERS = widgets/GraphGridLayout.h
+
 FORMS    += \
     dialogs/AboutDialog.ui \
     dialogs/preferences/AsmOptionsWidget.ui \

src/widgets/DisassemblerGraphView.cpp

@@ -103,6 +103,33 @@ DisassemblerGraphView::DisassemblerGraphView(QWidget *parent, CutterSeekable* se
 
     // Context menu that applies to everything
     contextMenu->addAction(&actionExportGraph);
+    static const std::pair<QString, GraphView::Layout> LAYOUT_CONFIG[] = {
+        {tr("Grid narrow"), GraphView::Layout::GridNarrow}
+        ,{tr("Grid medium"), GraphView::Layout::GridMedium}
+        ,{tr("Grid wide"), GraphView::Layout::GridWide}
+#ifdef CUTTER_ENABLE_GRAPHVIZ
+        ,{tr("Graphviz polyline"), GraphView::Layout::GraphvizPolyline}
+        ,{tr("Graphviz polyline LR"), GraphView::Layout::GraphvizPolylineLR}
+        ,{tr("Graphviz ortho"), GraphView::Layout::GraphvizOrtho}
+        ,{tr("Graphviz ortho LR"), GraphView::Layout::GraphvizOrthoLR}
+#endif
+    };
+    auto layoutMenu = contextMenu->addMenu(tr("Layout"));
+    QActionGroup* layoutGroup = new QActionGroup(layoutMenu);
+    for (auto &item : LAYOUT_CONFIG) {
+        auto action = layoutGroup->addAction(item.first);
+        action->setCheckable(true);
+        GraphView::Layout layout = item.second;
+        connect(action, &QAction::triggered, this, [this, layout]() {
+            setGraphLayout(layout);
+            refreshView();
+            onSeekChanged(this->seekable->getOffset()); // try to keep the view on current block
+        });
+        if (layout == getGraphLayout()) {
+            action->setChecked(true);
+        }
+    }
+    layoutMenu->addActions(layoutGroup->actions());
     contextMenu->addSeparator();
     contextMenu->addAction(&actionSyncOffset);
 

src/widgets/GraphLayout.h

@@ -11,6 +11,10 @@ public:
     struct GraphEdge {
         ut64 target;
         QPolygonF polyline;
+        enum ArrowDirection {
+            Down, Left, Up, Right, None
+        };
+        ArrowDirection arrow = ArrowDirection::Down;
 
         explicit GraphEdge(ut64 target): target(target) {}
     };
@@ -25,6 +29,7 @@ public:
         // Edges
         std::vector<GraphEdge> edges;
     };
+    using Graph = std::unordered_map<ut64, GraphBlock>;
 
     struct LayoutConfig {
         int block_vertical_margin = 40;
@@ -33,7 +38,7 @@ public:
 
     GraphLayout(const LayoutConfig &layout_config) : layoutConfig(layout_config) {}
     virtual ~GraphLayout() {}
-    virtual void CalculateLayout(std::unordered_map<ut64, GraphBlock> &blocks, ut64 entry, int &width,
+    virtual void CalculateLayout(Graph &blocks, ut64 entry, int &width,
                                  int &height) const = 0;
 protected:
     LayoutConfig layoutConfig;

src/widgets/GraphView.cpp

@@ -1,6 +1,9 @@
 #include "GraphView.h"
 
 #include "GraphGridLayout.h"
+#ifdef CUTTER_ENABLE_GRAPHVIZ
+#include "GraphvizLayout.h"
+#endif
 
 #include <vector>
 #include <QPainter>
@@ -17,7 +20,6 @@
 
 GraphView::GraphView(QWidget *parent)
     : QAbstractScrollArea(parent)
-    , graphLayoutSystem(new GraphGridLayout())
     , useGL(false)
 #ifndef QT_NO_OPENGL
     , cacheTexture(0)
@@ -32,6 +34,7 @@ GraphView::GraphView(QWidget *parent)
         glWidget = nullptr;
     }
 #endif
+    setGraphLayout(Layout::GridMedium);
 }
 
 GraphView::~GraphView()
@@ -349,22 +352,42 @@ void GraphView::paintGraphCache()
             p.drawPolyline(polyline);
             pen.setStyle(Qt::SolidLine);
             p.setPen(pen);
+
+            auto drawArrow = [&](QPointF tip, QPointF dir) {
+                QPolygonF arrow;
+                arrow << tip;
+                QPointF dy(-dir.y(), dir.x());
+                QPointF base = tip - dir * 6;
+                arrow << base + 3 * dy;
+                arrow << base - 3 * dy;
+                p.drawConvexPolygon(recalculatePolygon(arrow));
+            };
+
             if (!polyline.empty()) {
                 if (ec.start_arrow) {
                     auto firstPt = edge.polyline.first();
-                    QPolygonF arrowStart;
+                    drawArrow(firstPt, QPointF(0, 1));
-                    arrowStart << QPointF(firstPt.x() - 3, firstPt.y() + 6);
-                    arrowStart << QPointF(firstPt.x() + 3, firstPt.y() + 6);
-                    arrowStart << QPointF(firstPt);
-                    p.drawConvexPolygon(recalculatePolygon(arrowStart));
                 }
                 if (ec.end_arrow) {
                     auto lastPt = edge.polyline.last();
-                    QPolygonF arrowEnd;
+                    QPointF dir(0, -1);
-                    arrowEnd << QPointF(lastPt.x() - 3, lastPt.y() - 6);
+                    switch(edge.arrow) {
-                    arrowEnd << QPointF(lastPt.x() + 3, lastPt.y() - 6);
+                        case GraphLayout::GraphEdge::Down:
-                    arrowEnd << QPointF(lastPt);
+                            dir = QPointF(0, 1);
-                    p.drawConvexPolygon(recalculatePolygon(arrowEnd));
+                            break;
+                        case GraphLayout::GraphEdge::Up:
+                            dir = QPointF(0, -1);
+                            break;
+                        case GraphLayout::GraphEdge::Left:
+                            dir = QPointF(-1, 0);
+                            break;
+                        case GraphLayout::GraphEdge::Right:
+                            dir = QPointF(1, 0);
+                            break;
+                        default:
+                            break;
+                    }
+                    drawArrow(lastPt, dir);
                 }
             }
         }
@@ -443,6 +466,36 @@ void GraphView::showRectangle(const QRect &block, bool anywhere)
     viewport()->update();
 }
 
+void GraphView::setGraphLayout(GraphView::Layout layout)
+{
+    graphLayout = layout;
+    switch (layout) {
+        case Layout::GridNarrow:
+            this->graphLayoutSystem.reset(new GraphGridLayout(GraphGridLayout::LayoutType::Narrow));
+            break;
+        case Layout::GridMedium:
+            this->graphLayoutSystem.reset(new GraphGridLayout(GraphGridLayout::LayoutType::Medium));
+            break;
+        case Layout::GridWide:
+            this->graphLayoutSystem.reset(new GraphGridLayout(GraphGridLayout::LayoutType::Wide));
+            break;
+#ifdef CUTTER_ENABLE_GRAPHVIZ
+        case Layout::GraphvizOrtho:
+            this->graphLayoutSystem.reset(new GraphvizLayout(GraphvizLayout::LineType::Ortho));
+            break;
+        case Layout::GraphvizOrthoLR:
+            this->graphLayoutSystem.reset(new GraphvizLayout(GraphvizLayout::LineType::Ortho, GraphvizLayout::Direction::LR));
+            break;
+        case Layout::GraphvizPolyline:
+            this->graphLayoutSystem.reset(new GraphvizLayout(GraphvizLayout::LineType::Polyline));
+            break;
+        case Layout::GraphvizPolylineLR:
+            this->graphLayoutSystem.reset(new GraphvizLayout(GraphvizLayout::LineType::Polyline, GraphvizLayout::Direction::LR));
+            break;
+#endif
+    }
+}
+
 void GraphView::addBlock(GraphView::GraphBlock block)
 {
     blocks[block.entry] = block;

src/widgets/GraphView.h

@@ -33,6 +33,18 @@ public:
     using GraphBlock = GraphLayout::GraphBlock;
     using GraphEdge = GraphLayout::GraphEdge;
 
+    enum class Layout {
+        GridNarrow
+        ,GridMedium
+        ,GridWide
+#ifdef CUTTER_ENABLE_GRAPHVIZ
+        ,GraphvizOrtho
+        ,GraphvizOrthoLR
+        ,GraphvizPolyline
+        ,GraphvizPolylineLR
+#endif
+    };
+
     struct EdgeConfiguration {
         QColor color = QColor(128, 128, 128);
         bool start_arrow = false;
@@ -60,6 +72,9 @@ public:
      */
     ut64 currentFcnAddr = RVA_INVALID; // TODO: move application specific code out of graph view
 
+    void setGraphLayout(Layout layout);
+    Layout getGraphLayout() const { return graphLayout; }
+
 protected:
     std::unordered_map<ut64, GraphBlock> blocks;
     QColor backgroundColor = QColor(Qt::white);
@@ -140,6 +155,7 @@ private:
     QSize cacheSize;
     QOpenGLWidget *glWidget;
 #endif
+    Layout graphLayout;
 
     /**
      * @brief flag to control if the cache is invalid and should be re-created in the next draw

src/widgets/GraphvizLayout.cpp

@@ -0,0 +1,215 @@
+#include "GraphvizLayout.h"
+
+#include <unordered_set>
+#include <unordered_map>
+#include <queue>
+#include <stack>
+#include <cassert>
+#include <sstream>
+#include <iomanip>
+#include <set>
+
+#include <gvc.h>
+
+GraphvizLayout::GraphvizLayout(LineType lineType, Direction direction)
+    : GraphLayout({})
+    , direction(direction)
+    , lineType(lineType)
+{
+}
+
+static GraphLayout::GraphEdge::ArrowDirection getArrowDirection(QPointF direction,
+                                                                bool preferVertical)
+{
+    if (abs(direction.x()) > abs(direction.y()) * (preferVertical ? 3.0 : 1.0)) {
+        if (direction.x() > 0) {
+            return GraphLayout::GraphEdge::Right;
+        } else {
+            return GraphLayout::GraphEdge::Left;
+        }
+    } else {
+        if (direction.y() > 0) {
+            return GraphLayout::GraphEdge::Down;
+        } else {
+            return GraphLayout::GraphEdge::Up;
+        }
+    }
+}
+
+static std::set<std::pair<ut64, ut64>> SelectLoopEdges(const GraphLayout::Graph &graph, ut64 entry)
+{
+    std::set<std::pair<ut64, ut64>> result;
+    // Run DFS to select backwards/loop edges
+    // 0 - not visited
+    // 1 - in stack
+    // 2 - visited
+    std::unordered_map<ut64, uint8_t> visited;
+    visited.reserve(graph.size());
+    std::stack<std::pair<ut64, size_t>> stack;
+    auto dfsFragment = [&visited, &graph, &stack, &result](ut64 first) {
+        visited[first] = 1;
+        stack.push({first, 0});
+        while (!stack.empty()) {
+            auto v = stack.top().first;
+            auto edge_index = stack.top().second;
+            auto blockIt = graph.find(v);
+            if (blockIt == graph.end()) {
+                continue;
+            }
+            const auto &block = blockIt->second;
+            if (edge_index < block.edges.size()) {
+                ++stack.top().second;
+                auto target = block.edges[edge_index].target;
+                auto &targetState = visited[target];
+                if (targetState == 0) {
+                    targetState = 1;
+                    stack.push({target, 0});
+                } else if (targetState == 1) {
+                    result.insert({v, target});
+                }
+            } else {
+                stack.pop();
+                visited[v] = 2;
+            }
+        }
+    };
+
+    dfsFragment(entry);
+    for (auto &blockIt : graph) {
+        if (!visited[blockIt.first]) {
+            dfsFragment(blockIt.first);
+        }
+    }
+
+    return result;
+}
+
+void GraphvizLayout::CalculateLayout(std::unordered_map<ut64, GraphBlock> &blocks, ut64 entry,
+                                     int &width, int &height) const
+{
+    //https://gitlab.com/graphviz/graphviz/issues/1441
+#define STR(v) const_cast<char*>(v)
+
+    width = height = 10;
+    GVC_t *gvc = gvContext();
+    Agraph_t *g = agopen(STR("G"), Agdirected, nullptr);
+
+    std::unordered_map<ut64, Agnode_t *> nodes;
+    for (const auto &block : blocks) {
+        nodes[block.first] = agnode(g, nullptr, TRUE);
+    }
+
+    std::vector<std::string> strc;
+    strc.reserve(2 * blocks.size());
+    std::map<std::pair<ut64, ut64>, Agedge_t *> edges;
+
+    agsafeset(g, STR("splines"), lineType == LineType::Ortho ? STR("ortho") : STR("polyline"), STR(""));
+    switch (direction) {
+    case Direction::LR:
+        agsafeset(g, STR("rankdir"), STR("LR"), STR(""));
+        break;
+    case Direction::TB:
+        agsafeset(g, STR("rankdir"), STR("BT"), STR(""));
+        break;
+    }
+    agsafeset(g, STR("newrank"), STR("true"), STR(""));
+    // graphviz has builtin 72 dpi setting for input that differs from output
+    // it's easier to use 72 everywhere
+    const double dpi = 72.0;
+    agsafeset(g, STR("dpi"), STR("72"), STR(""));
+
+    auto widhAttr = agattr(g, AGNODE, STR("width"), STR("1"));
+    auto heightAatr = agattr(g, AGNODE, STR("height"), STR("1"));
+    agattr(g, AGNODE, STR("shape"), STR("box"));
+    agattr(g, AGNODE, STR("fixedsize"), STR("true"));
+    auto constraintAttr = agattr(g, AGEDGE, STR("constraint"), STR("1"));
+
+    std::ostringstream stream;
+    stream.imbue(std::locale::classic());
+    auto setFloatingPointAttr = [&stream](void *obj, Agsym_t *sym, double value) {
+        stream.str({});
+        stream << std::fixed << std::setw(4) << value;
+        auto str = stream.str();
+        agxset(obj, sym, STR(str.c_str()));
+    };
+
+    std::set<std::pair<ut64, ut64>> loopEdges = SelectLoopEdges(blocks, entry);
+
+    for (const auto &blockIt : blocks) {
+        auto u = nodes[blockIt.first];
+        auto &block = blockIt.second;
+
+        for (auto &edge : block.edges) {
+            auto v = nodes.find(edge.target);
+            if (v == nodes.end()) {
+                continue;
+            }
+            auto e = agedge(g, u, v->second, nullptr, TRUE);
+            edges[{blockIt.first, edge.target}] = e;
+            if (loopEdges.find({blockIt.first, edge.target}) != loopEdges.end()) {
+                agxset(e, constraintAttr, STR("0"));
+            }
+        }
+        setFloatingPointAttr(u, widhAttr, block.width / dpi);
+        setFloatingPointAttr(u, heightAatr, block.height / dpi);
+    }
+
+    gvLayout(gvc, g, "dot");
+
+    for (auto &blockIt : blocks) {
+        auto &block = blockIt.second;
+        auto u = nodes[blockIt.first];
+
+        auto pos = ND_coord(u);
+
+        auto w = ND_width(u) * dpi;
+        auto h = ND_height(u) * dpi;
+        block.x = pos.x  - w / 2.0;
+        block.y = pos.y  - h / 2.0;
+        width = std::max(width, block.x + block.width);
+        height = std::max(height, block.y + block.height);
+
+        for (auto &edge : block.edges) {
+            auto it = edges.find({blockIt.first, edge.target});
+            if (it != edges.end()) {
+                auto e = it->second;
+                if (auto spl = ED_spl(e)) {
+                    for (int i = 0; i < 1 && i < spl->size; i++) {
+                        auto bz = spl->list[i];
+                        edge.polyline.reserve(bz.size + 1);
+                        for (int j = 0; j < bz.size; j++) {
+                            edge.polyline.push_back(QPointF(bz.list[j].x, bz.list[j].y));
+                        }
+                        QPointF last(0, 0);
+                        if (!edge.polyline.empty()) {
+                            last = edge.polyline.back();
+                        }
+                        if (bz.eflag) {
+                            QPointF tip = QPointF(bz.ep.x, bz.ep.y);
+                            edge.polyline.push_back(tip);
+                        }
+
+                        if (edge.polyline.size() >= 2) {
+                            // make sure self loops go from bottom to top
+                            if (edge.target == block.entry && edge.polyline.first().y() < edge.polyline.last().y()) {
+                                std::reverse(edge.polyline.begin(), edge.polyline.end());
+                            }
+                            auto it = edge.polyline.rbegin();
+                            QPointF direction = *it;
+                            direction -= *(++it);
+                            edge.arrow = getArrowDirection(direction, lineType == LineType::Polyline);
+
+                        } else {
+                            edge.arrow = GraphEdge::Down;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    gvFreeLayout(gvc, g);
+    agclose(g);
+    gvFreeContext(gvc);
+#undef STR
+}

src/widgets/GraphvizLayout.h

@@ -0,0 +1,28 @@
+#ifndef GRAPHVIZLAYOUT_H
+#define GRAPHVIZLAYOUT_H
+
+#include "core/Cutter.h"
+#include "GraphLayout.h"
+
+class GraphvizLayout : public GraphLayout
+{
+public:
+    enum class LineType {
+        Ortho,
+        Polyline
+    };
+    enum class Direction {
+        TB,
+        LR
+    };
+    GraphvizLayout(LineType lineType, Direction direction = Direction::TB);
+    virtual void CalculateLayout(std::unordered_map<ut64, GraphBlock> &blocks,
+                                 ut64 entry,
+                                 int &width,
+                                 int &height) const override;
+private:
+    Direction direction;
+    LineType lineType;
+};
+
+#endif // GRAPHVIZLAYOUT_H