From ef72596436bdf090d82738adb15f6212c6025961 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 16:45:27 +0200 Subject: [PATCH 1/9] Add nodejs support --- CMakeLists.txt | 1 + cmake/compiler-env.cmake | 20 +++- cmake/misc/node-pre-script.js | 3 + deps/unicorn | 2 +- .../unicorn-emulator/function_wrapper_tcg.hpp | 91 ------------------- .../unicorn-emulator/unicorn_x64_emulator.cpp | 3 +- 6 files changed, 22 insertions(+), 98 deletions(-) create mode 100644 cmake/misc/node-pre-script.js delete mode 100644 src/backends/unicorn-emulator/function_wrapper_tcg.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f430a56c..ec3e355d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ option(MOMO_ENABLE_AVX2 "Enable AVX2 support" ON) option(MOMO_ENABLE_SANITIZER "Enable sanitizer" OFF) option(MOMO_ENABLE_CLANG_TIDY "Enable clang-tidy checks" OFF) option(MOMO_ENABLE_RUST_CODE "Enable code parts written in rust" ON) +option(MOMO_EMSCRIPTEN_SUPPORT_NODEJS "Enable Node.js filesystem for emscripten compilation" OFF) option(MOMO_BUILD_AS_LIBRARY "Configure and Build the emulator as a shared library (without the samples and tests)" OFF) set(MOMO_REFLECTION_LEVEL "0" CACHE STRING "Reflection level for the build") diff --git a/cmake/compiler-env.cmake b/cmake/compiler-env.cmake index 77951879..d9d09c5c 100644 --- a/cmake/compiler-env.cmake +++ b/cmake/compiler-env.cmake @@ -89,8 +89,9 @@ endif() ########################################## if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") - add_compile_options( + momo_add_c_and_cxx_compile_options( -fexceptions + -ftrivial-auto-var-init=zero ) add_link_options( @@ -98,13 +99,24 @@ if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") -sALLOW_MEMORY_GROWTH=1 -sASSERTIONS -sWASM_BIGINT - -sENVIRONMENT=web -sUSE_OFFSET_CONVERTER #-sEXCEPTION_CATCHING_ALLOWED=[..] -sEXIT_RUNTIME - #-lnodefs.js -sNODERAWFS=1 #-sASYNCIFY -) + ) + + if(MOMO_EMSCRIPTEN_SUPPORT_NODEJS) + add_link_options( + -lnodefs.js -sNODERAWFS=1 + -sENVIRONMENT=node + -sMAXIMUM_MEMORY=4gb + --pre-js ${CMAKE_CURRENT_LIST_DIR}/misc/node-pre-script.js + ) + else() + add_link_options( + -sENVIRONMENT=worker + ) + endif() endif() ########################################## diff --git a/cmake/misc/node-pre-script.js b/cmake/misc/node-pre-script.js new file mode 100644 index 00000000..ee8eb4ee --- /dev/null +++ b/cmake/misc/node-pre-script.js @@ -0,0 +1,3 @@ +Module['preRun'] = () => { + ENV = process.env; +}; diff --git a/deps/unicorn b/deps/unicorn index 6b329651..843eb15b 160000 --- a/deps/unicorn +++ b/deps/unicorn @@ -1 +1 @@ -Subproject commit 6b32965144d4ff91eb303beed0ec938ad467f33b +Subproject commit 843eb15bcd01571a5062fa0f3937fc8e49a40cdc diff --git a/src/backends/unicorn-emulator/function_wrapper_tcg.hpp b/src/backends/unicorn-emulator/function_wrapper_tcg.hpp deleted file mode 100644 index 4cdaed8a..00000000 --- a/src/backends/unicorn-emulator/function_wrapper_tcg.hpp +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -T resolve_indexed_argument_internal(const std::array& args, size_t& index) -{ - const auto a1 = args[index++]; - - if constexpr (sizeof(T) <= sizeof(a1) || sizeof(size_t) > 4) - { - return T(a1); - } - else - { - const auto a2 = args[index++]; - - const auto arg = (a1 | (static_cast(a2) << 32)); - return T(arg); - } -} - -template -T resolve_indexed_argument(const std::array& args, size_t& index) -{ - auto arg = resolve_indexed_argument_internal(args, index); - return arg; -} - -template -class function_wrapper_tcg : public utils::object -{ - public: - using user_data_pointer = void*; - using c_function_type = ReturnType(Args..., user_data_pointer); - using functor_type = std::function; - - function_wrapper_tcg() = default; - - function_wrapper_tcg(functor_type functor) - : functor_(std::make_unique(std::move(functor))) - { - } - - c_function_type* get_c_function() const - { - auto* func = +[](const size_t a1, const size_t a2, const size_t a3, const size_t a4, const size_t a5, - const size_t a6, const size_t a7, const size_t a8, const size_t a9, const size_t a10, - const size_t a11, const size_t a12) -> uint64_t { - const std::array arguments = {a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12}; - - const auto lambda = +[](Args... args, user_data_pointer user_data) -> ReturnType { - return (*static_cast(user_data))(std::forward(args)...); - }; - - size_t index = 0; - std::tuple func_args{ - resolve_indexed_argument>>(arguments, index)..., - resolve_indexed_argument(arguments, index)}; - - (void)index; - - if constexpr (!std::is_void_v) - { - return uint64_t(std::apply(lambda, std::move(func_args))); - } - - std::apply(lambda, std::move(func_args)); - return 0; - }; - - return reinterpret_cast(reinterpret_cast(func)); - } - - void* get_function() const - { - return reinterpret_cast(this->get_c_function()); - } - - user_data_pointer get_user_data() const - { - return this->functor_.get(); - } - - private: - std::unique_ptr functor_{}; -}; diff --git a/src/backends/unicorn-emulator/unicorn_x64_emulator.cpp b/src/backends/unicorn-emulator/unicorn_x64_emulator.cpp index 5be03b90..d542ed8f 100644 --- a/src/backends/unicorn-emulator/unicorn_x64_emulator.cpp +++ b/src/backends/unicorn-emulator/unicorn_x64_emulator.cpp @@ -7,7 +7,6 @@ #include "unicorn_hook.hpp" #include "function_wrapper.hpp" -#include "function_wrapper_tcg.hpp" #include namespace unicorn @@ -550,7 +549,7 @@ namespace unicorn c(address); // }; - function_wrapper_tcg wrapper(std::move(exec_wrapper)); + function_wrapper wrapper(std::move(exec_wrapper)); unicorn_hook hook{*this}; From 67534393ff364fb537812a9be19f8f5fdd009d73 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 16:59:13 +0200 Subject: [PATCH 2/9] Build and test nodejs --- .github/workflows/build.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6860204..cd1b0a94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -161,7 +161,8 @@ jobs: - iOS arm64 - Android x86_64 - Android arm64-v8a - - Emscripten + - Emscripten Web + - Emscripten Node.js configuration: - Debug - Release @@ -202,9 +203,12 @@ jobs: abi: arm64-v8a rust-target: aarch64-linux-android cmake-options: "-DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/cmake/toolchain/android-ndk.cmake" - - platform: Emscripten + - platform: Emscripten Web runner: ubuntu-24.04 cmake-options: "-DMOMO_ENABLE_RUST_CODE=Off -DCMAKE_TOOLCHAIN_FILE=$(dirname $(which emcc))/cmake/Modules/Platform/Emscripten.cmake" + - platform: Emscripten Node.js + runner: ubuntu-24.04 + cmake-options: "-DMOMO_EMSCRIPTEN_SUPPORT_NODEJS=On -DMOMO_ENABLE_RUST_CODE=Off -DCMAKE_TOOLCHAIN_FILE=$(dirname $(which emcc))/cmake/Modules/Platform/Emscripten.cmake" steps: - name: Checkout Source uses: actions/checkout@v4 @@ -219,7 +223,7 @@ jobs: run: rustup target add ${{ matrix.rust-target }} - name: Install Emscripten - if: "${{ matrix.platform == 'Emscripten' }}" + if: "${{ startsWith(matrix.platform, 'Emscripten') }}" uses: mymindstorm/setup-emsdk@v14 - name: Install Clang @@ -284,6 +288,7 @@ jobs: - Linux x86_64 Clang - macOS arm64 - macOS x86_64 + - Emscripten Node.js emulator: - Unicorn - Icicle @@ -313,6 +318,8 @@ jobs: runner: macos-latest - platform: macOS x86_64 runner: macos-13 + - platform: Emscripten Node.js + runner: ubuntu-24.04 steps: - name: Checkout Source uses: actions/checkout@v4 @@ -354,7 +361,7 @@ jobs: run: cp build/${{matrix.preset}}/artifacts/test-sample.exe build/${{matrix.preset}}/artifacts/root/filesys/c/ - name: CMake Test - if: ${{ matrix.emulator != 'Icicle' || matrix.platform != 'Windows x86' }} + if: ${{ matrix.emulator != 'Icicle' || (matrix.platform != 'Windows x86' && !startsWith(matrix.platform, 'Emscripten')) }} run: cd build/${{matrix.preset}} && ctest --verbose -j env: EMULATOR_ROOT: ${{github.workspace}}/build/${{matrix.preset}}/artifacts/root @@ -495,10 +502,10 @@ jobs: with: submodules: recursive - - name: Download Emscripten Artifacts + - name: Download Emscripten Web Artifacts uses: pyTooling/download-artifact@v4 with: - name: Emscripten Release Artifacts + name: Emscripten Web Release Artifacts path: build/release/artifacts - name: Download Windows Artifacts From ff050ed74ce1863fe0944a125beaa46443eeb42d Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:17:15 +0200 Subject: [PATCH 3/9] Fix node tests --- src/analyzer/CMakeLists.txt | 8 +++++++- src/analyzer/test.py | 9 +++++++++ src/windows-emulator-test/CMakeLists.txt | 8 +++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/analyzer/CMakeLists.txt b/src/analyzer/CMakeLists.txt index 0cfffb61..7e40d529 100644 --- a/src/analyzer/CMakeLists.txt +++ b/src/analyzer/CMakeLists.txt @@ -32,6 +32,12 @@ set(ENV_PREFIX "%") set(ENV_SUFFIX "%") endif() +set(ANYLZER_TEST_ENVIRONMENT "default") + +if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") + set(ANYLZER_TEST_ENVIRONMENT "node") +endif() + add_test(NAME analyzer-test - COMMAND "${PYTHON3_EXE}" "${CMAKE_CURRENT_LIST_DIR}/test.py" + COMMAND "${PYTHON3_EXE}" "${CMAKE_CURRENT_LIST_DIR}/test.py" "${ANYLZER_TEST_ENVIRONMENT}" WORKING_DIRECTORY "$") diff --git a/src/analyzer/test.py b/src/analyzer/test.py index fd907dc4..671c54e0 100644 --- a/src/analyzer/test.py +++ b/src/analyzer/test.py @@ -1,4 +1,5 @@ import os +import sys import subprocess emulator_root = os.getenv('EMULATOR_ROOT') @@ -7,7 +8,12 @@ virtual_sample = 'C:/analysis-sample.exe' application = 'analyzer' +is_node = len(sys.argv) > 1 and sys.argv == "node" + def make_app(app): + if is_node: + return app + ".js" + if os.name == 'nt': return app + ".exe" @@ -21,6 +27,9 @@ command = [ virtual_sample ] +if is_node: + command = ["node"] + command + result = subprocess.run(command, cwd=os.getcwd()) exit(result.returncode) diff --git a/src/windows-emulator-test/CMakeLists.txt b/src/windows-emulator-test/CMakeLists.txt index 8bea4b26..91ed881d 100644 --- a/src/windows-emulator-test/CMakeLists.txt +++ b/src/windows-emulator-test/CMakeLists.txt @@ -20,8 +20,14 @@ if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8) add_dependencies(windows-emulator-test test-sample) endif() +set(TEST_COMMAND "./windows-emulator-test") + +if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") + set(TEST_COMMAND node "./windows-emulator-test.js") +endif() + add_test(NAME windows-emulator-test - COMMAND "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ./windows-emulator-test + COMMAND "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ${TEST_COMMAND} WORKING_DIRECTORY "$") momo_targets_set_folder("tests" windows-emulator-test) From 59f700e1cd6a10b9d3750d6545e5ab41c35e4ae3 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:28:42 +0200 Subject: [PATCH 4/9] Revert "Fix node tests" This reverts commit ff050ed74ce1863fe0944a125beaa46443eeb42d. --- src/analyzer/CMakeLists.txt | 8 +------- src/analyzer/test.py | 9 --------- src/windows-emulator-test/CMakeLists.txt | 8 +------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/analyzer/CMakeLists.txt b/src/analyzer/CMakeLists.txt index 7e40d529..0cfffb61 100644 --- a/src/analyzer/CMakeLists.txt +++ b/src/analyzer/CMakeLists.txt @@ -32,12 +32,6 @@ set(ENV_PREFIX "%") set(ENV_SUFFIX "%") endif() -set(ANYLZER_TEST_ENVIRONMENT "default") - -if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") - set(ANYLZER_TEST_ENVIRONMENT "node") -endif() - add_test(NAME analyzer-test - COMMAND "${PYTHON3_EXE}" "${CMAKE_CURRENT_LIST_DIR}/test.py" "${ANYLZER_TEST_ENVIRONMENT}" + COMMAND "${PYTHON3_EXE}" "${CMAKE_CURRENT_LIST_DIR}/test.py" WORKING_DIRECTORY "$") diff --git a/src/analyzer/test.py b/src/analyzer/test.py index 671c54e0..fd907dc4 100644 --- a/src/analyzer/test.py +++ b/src/analyzer/test.py @@ -1,5 +1,4 @@ import os -import sys import subprocess emulator_root = os.getenv('EMULATOR_ROOT') @@ -8,12 +7,7 @@ virtual_sample = 'C:/analysis-sample.exe' application = 'analyzer' -is_node = len(sys.argv) > 1 and sys.argv == "node" - def make_app(app): - if is_node: - return app + ".js" - if os.name == 'nt': return app + ".exe" @@ -27,9 +21,6 @@ command = [ virtual_sample ] -if is_node: - command = ["node"] + command - result = subprocess.run(command, cwd=os.getcwd()) exit(result.returncode) diff --git a/src/windows-emulator-test/CMakeLists.txt b/src/windows-emulator-test/CMakeLists.txt index 91ed881d..8bea4b26 100644 --- a/src/windows-emulator-test/CMakeLists.txt +++ b/src/windows-emulator-test/CMakeLists.txt @@ -20,14 +20,8 @@ if(WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8) add_dependencies(windows-emulator-test test-sample) endif() -set(TEST_COMMAND "./windows-emulator-test") - -if(CMAKE_SYSTEM_NAME MATCHES "Emscripten") - set(TEST_COMMAND node "./windows-emulator-test.js") -endif() - add_test(NAME windows-emulator-test - COMMAND "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ${TEST_COMMAND} + COMMAND "${PYTHON3_EXE}" "${PROJECT_SOURCE_DIR}/deps/gtest-parallel/gtest_parallel.py" ./windows-emulator-test WORKING_DIRECTORY "$") momo_targets_set_folder("tests" windows-emulator-test) From 9f10a555b085796d8aafb2e263c5d969789511b3 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:33:07 +0200 Subject: [PATCH 5/9] Add nodejs smoke test --- .github/workflows/build.yml | 43 ++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd1b0a94..b4595d51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -288,7 +288,6 @@ jobs: - Linux x86_64 Clang - macOS arm64 - macOS x86_64 - - Emscripten Node.js emulator: - Unicorn - Icicle @@ -361,7 +360,7 @@ jobs: run: cp build/${{matrix.preset}}/artifacts/test-sample.exe build/${{matrix.preset}}/artifacts/root/filesys/c/ - name: CMake Test - if: ${{ matrix.emulator != 'Icicle' || (matrix.platform != 'Windows x86' && !startsWith(matrix.platform, 'Emscripten')) }} + if: ${{ matrix.emulator != 'Icicle' || matrix.platform != 'Windows x86' }} run: cd build/${{matrix.preset}} && ctest --verbose -j env: EMULATOR_ROOT: ${{github.workspace}}/build/${{matrix.preset}}/artifacts/root @@ -423,6 +422,44 @@ jobs: ANALYSIS_SAMPLE: ${{github.workspace}}/build/release/artifacts/test-sample.exe + smoke-test-node: + name: Smoke Test Node.js + runs-on: ubuntu-24.04 + needs: [create-emulation-root, build] + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Download Artifacts + uses: pyTooling/download-artifact@v4 + with: + name: Emscripten Node.js Release Artifacts + path: build/release/artifacts + + - name: Download Windows Artifacts + uses: pyTooling/download-artifact@v4 + with: + name: Windows x86_64 Release Artifacts + path: build/release/artifacts + + - name: Download Emulation Root + uses: pyTooling/download-artifact@v4 + with: + name: Windows 2022 Emulation Root + path: build/release/artifacts/root + + - name: Copy Test Sample + run: cp build/release/artifacts/test-sample.exe build/release/artifacts/root/filesys/c/ + + - name: CMake Test + run: cd build/release/artifacts && node ./windows-emulator-test.js + env: + EMULATOR_ROOT: ${{github.workspace}}/build/release/artifacts/root + EMULATOR_VERBOSE: ${{ github.event.inputs.verbose }} + + smoke-test-android: name: Smoke Test Android runs-on: ${{ matrix.runner }} @@ -569,7 +606,7 @@ jobs: summary: name: Pipeline Summary runs-on: ubuntu-24.04 - needs: [build-page, clang-tidy, build-apiset-dumper, smoke-test-android, create-emulation-root, build, test, win-test, verify-formatting] + needs: [build-page, clang-tidy, build-apiset-dumper, smoke-test-node, smoke-test-android, create-emulation-root, build, test, win-test, verify-formatting] if: always() steps: - uses: geekyeggo/delete-artifact@v5 From 38b2c05fa720cb919690b8ff843ff19be5355df0 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:37:26 +0200 Subject: [PATCH 6/9] Maybe fix unicorn --- deps/unicorn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/unicorn b/deps/unicorn index 843eb15b..8dc3f4c5 160000 --- a/deps/unicorn +++ b/deps/unicorn @@ -1 +1 @@ -Subproject commit 843eb15bcd01571a5062fa0f3937fc8e49a40cdc +Subproject commit 8dc3f4c5cd6d8053428c3384baf0710f0ee60b40 From 26afe5731c9afd90f26079b1726b825f479c89ae Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:42:11 +0200 Subject: [PATCH 7/9] Fix pipeline --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4595d51..3219cd96 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -317,8 +317,6 @@ jobs: runner: macos-latest - platform: macOS x86_64 runner: macos-13 - - platform: Emscripten Node.js - runner: ubuntu-24.04 steps: - name: Checkout Source uses: actions/checkout@v4 From a157a76cae7e68ee44ab40abfaf735b6cd8552c4 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 17:54:59 +0200 Subject: [PATCH 8/9] Ensure bash is used to set environment variables --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3219cd96..00455b6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -252,6 +252,7 @@ jobs: add-to-path: false - name: Setup Environment Variables + shell: bash if: ${{ startsWith(matrix.platform, 'Android') }} run: | echo "ANDROID_NDK_ROOT=${{ steps.setup-ndk.outputs.ndk-path }}" >> $GITHUB_ENV @@ -324,6 +325,7 @@ jobs: submodules: recursive - name: Setup Environment Variables + shell: bash run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV echo "ASAN_OPTIONS=detect_odr_violation=0" >> $GITHUB_ENV @@ -386,6 +388,7 @@ jobs: submodules: recursive - name: Setup Environment Variables + shell: bash run: | echo "RUST_BACKTRACE=1" >> $GITHUB_ENV echo "ASAN_OPTIONS=detect_odr_violation=0" >> $GITHUB_ENV From fc48ce11027b3aea345415b365d28dc8503c262c Mon Sep 17 00:00:00 2001 From: momo5502 Date: Fri, 18 Apr 2025 18:19:39 +0200 Subject: [PATCH 9/9] Update unicorn --- deps/unicorn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/unicorn b/deps/unicorn index 8dc3f4c5..a19ae94c 160000 --- a/deps/unicorn +++ b/deps/unicorn @@ -1 +1 @@ -Subproject commit 8dc3f4c5cd6d8053428c3384baf0710f0ee60b40 +Subproject commit a19ae94cbf42ddb6aa2bf0b60c7c0fa907af039b