Add capstone disassembler (#448)

.gitmodules

@@ -33,4 +33,7 @@
 	path = deps/minidump_cpp
 	url = https://github.com/redthing1/minidump_cpp
 	shallow = true
-
+[submodule "deps/capstone"]
+	path = deps/capstone
+	url = https://github.com/capstone-engine/capstone.git
+	shallow = true

deps/CMakeLists.txt

@@ -29,5 +29,15 @@ add_subdirectory(minidump_cpp)
 
 ##########################################
 
+
+option(CAPSTONE_BUILD_MACOS_THIN "" ON)
+option(CAPSTONE_X86_SUPPORT "" ON)
+option(CAPSTONE_X86_ATT_DISABLE "" ON)
+option(CAPSTONE_ARCHITECTURE_DEFAULT "" OFF)
+option(CAPSTONE_BUILD_STATIC_MSVC_RUNTIME "" OFF)
+add_subdirectory(capstone)
+
+##########################################
+
 include(googletest.cmake)
 include(zlib.cmake)

src/analyzer/CMakeLists.txt

@@ -17,6 +17,7 @@ endif()
 target_link_libraries(analyzer PRIVATE
   reflect
   debugger
+  capstone
   windows-emulator
   windows-gdb-stub
   backend-selection

src/analyzer/analysis.cpp

@@ -1,6 +1,7 @@
 #include "std_include.hpp"
 
 #include "analysis.hpp"
+#include "disassembler.hpp"
 #include "windows_emulator.hpp"
 #include <utils/lazy_object.hpp>
 
@@ -98,7 +99,7 @@ namespace
 
     void handle_thread_set_name(const analysis_context& c, const emulator_thread& t)
     {
-        c.win_emu->log.print(color::blue, "Setting thread (%d) name: %s\n", t.id, u16_to_u8(t.name).c_str());
+        c.win_emu->log.print(color::blue, "Setting thread (%u) name: %s\n", t.id, u16_to_u8(t.name).c_str());
     }
 
     void handle_thread_switch(const analysis_context& c, const emulator_thread& current_thread,
@@ -149,7 +150,7 @@ namespace
         }
     }
 
-    void handle_function_details(analysis_context& c, const std::string_view function)
+    void handle_function_details(const analysis_context& c, const std::string_view function)
     {
         if (function == "GetEnvironmentVariableA" || function == "ExpandEnvironmentStringsA")
         {
@@ -234,6 +235,25 @@ namespace
         }
     }
 
+    bool is_return(const emulator& emu, const uint64_t address)
+    {
+        std::vector<uint8_t> instruction_bytes(15, 0);
+        const auto result = emu.try_read_memory(address, instruction_bytes.data(), instruction_bytes.size());
+        if (!result)
+        {
+            return false;
+        }
+
+        disassembler disasm{};
+        const auto instructions = disasm.disassemble(instruction_bytes, 1);
+        if (instructions.empty())
+        {
+            return false;
+        }
+
+        return cs_insn_group(disasm.get_handle(), &instructions[0], CS_GRP_RET);
+    }
+
     void handle_instruction(analysis_context& c, const uint64_t address)
     {
         auto& win_emu = *c.win_emu;
@@ -313,7 +333,8 @@ namespace
             win_emu.log.print(is_interesting_call ? color::yellow : color::gray,
                               "Executing entry point: %s (0x%" PRIx64 ")\n", binary->name.c_str(), address);
         }
-        else if (is_previous_main_exe && binary != previous_binary)
+        else if (is_previous_main_exe && binary != previous_binary &&
+                 !is_return(c.win_emu->emu(), win_emu.process.previous_ip))
         {
             auto nearest_entry = binary->address_names.upper_bound(address);
             if (nearest_entry == binary->address_names.begin())

src/analyzer/disassembler.cpp

@@ -0,0 +1,60 @@
+#include "std_include.hpp"
+
+#include "disassembler.hpp"
+
+disassembler::disassembler()
+{
+    const auto res = cs_open(CS_ARCH_X86, CS_MODE_64, &this->handle_);
+    if (res != CS_ERR_OK)
+    {
+        throw std::runtime_error("Failed to initialize capstone");
+    }
+}
+
+disassembler::~disassembler()
+{
+    this->release();
+}
+
+disassembler::disassembler(disassembler&& obj) noexcept
+{
+    this->operator=(std::move(obj));
+}
+
+disassembler& disassembler::operator=(disassembler&& obj) noexcept
+{
+    if (this != &obj)
+    {
+        this->release();
+        this->handle_ = obj.handle_;
+        obj.handle_ = 0;
+    }
+
+    return *this;
+}
+
+void disassembler::release()
+{
+    if (this->handle_)
+    {
+        cs_close(&this->handle_);
+        this->handle_ = 0;
+    }
+}
+
+instructions disassembler::disassemble(const std::span<const uint8_t> data, const size_t count) const
+{
+    cs_insn* insts{};
+    const auto inst_count = cs_disasm(this->handle_, data.data(), data.size(), count, 0, &insts);
+    return instructions{std::span(insts, inst_count)};
+}
+
+void instructions::release()
+{
+    if (!this->instructions_.empty())
+    {
+        cs_free(this->instructions_.data(), this->instructions_.size());
+    }
+
+    this->instructions_ = {};
+}

src/analyzer/disassembler.hpp

@@ -0,0 +1,105 @@
+#pragma once
+
+#include <capstone/capstone.h>
+#include <utils/finally.hpp>
+
+class instructions
+{
+  public:
+    instructions() = default;
+    ~instructions()
+    {
+        this->release();
+    }
+
+    instructions(instructions&& obj) noexcept
+        : instructions()
+    {
+        this->operator=(std::move(obj));
+    }
+
+    instructions& operator=(instructions&& obj) noexcept
+    {
+        if (this != &obj)
+        {
+            this->release();
+            this->instructions_ = obj.instructions_;
+            obj.instructions_ = {};
+        }
+
+        return *this;
+    }
+
+    instructions(const instructions&) = delete;
+    instructions& operator=(const instructions&) = delete;
+
+    operator std::span<cs_insn>() const
+    {
+        return this->instructions_;
+    }
+
+    bool empty() const noexcept
+    {
+        return this->instructions_.empty();
+    }
+
+    size_t size() const noexcept
+    {
+        return this->instructions_.size();
+    }
+
+    const cs_insn* data() const noexcept
+    {
+        return this->instructions_.data();
+    }
+
+    const cs_insn& operator[](const size_t index) const
+    {
+        return this->instructions_[index];
+    }
+
+    auto begin() const
+    {
+        return this->instructions_.begin();
+    }
+    auto end() const
+    {
+        return this->instructions_.end();
+    }
+
+  private:
+    friend class disassembler;
+    std::span<cs_insn> instructions_{};
+
+    explicit instructions(const std::span<cs_insn> insts)
+        : instructions_(insts)
+    {
+    }
+
+    void release();
+};
+
+class disassembler
+{
+  public:
+    disassembler();
+    ~disassembler();
+
+    disassembler(disassembler&& obj) noexcept;
+    disassembler& operator=(disassembler&& obj) noexcept;
+
+    disassembler(const disassembler& obj) = delete;
+    disassembler& operator=(const disassembler& obj) = delete;
+
+    instructions disassemble(std::span<const uint8_t> data, size_t count) const;
+
+    csh get_handle() const
+    {
+        return this->handle_;
+    }
+
+  private:
+    csh handle_{};
+
+    void release();
+};

src/windows-emulator-test/CMakeLists.txt

@@ -22,7 +22,7 @@ if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8)
 endif()
 
 add_test(NAME windows-emulator-test
-         COMMAND "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ./windows-emulator-test
+         COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=." -- "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ./windows-emulator-test
          WORKING_DIRECTORY "$<TARGET_FILE_DIR:windows-emulator-test>")
 
 momo_targets_set_folder("tests" windows-emulator-test)