54 cross platform api set support (#75)

This PR provides configurable APISET data for the PEB

There are multiple ways the APISET data can be provided:

* host -> read the APISET from the host process
* file -> read the APISET from an external file
* w11  -> use an inbuilt dump from windows 11
* w10  -> use an inbuilt dump from windows 10

by default on windows it uses the "host" mode.
by default on linux/mac it uses the "w11" mode.

with the file mode you can use the dump-apiset tool to create a dump.
however the inbuilt w10 works fine on w11 so I don't expect it will be
needed.

.github/workflows/build.yml

@@ -36,7 +36,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Dump Registry
-        run: src/grab-registry.bat
+        run: src/tools/grab-registry.bat
 
       - name: Upload Artifacts
         uses: actions/upload-artifact@v4

.gitmodules

@@ -14,3 +14,8 @@
 [submodule "deps/googletest"]
 	path = deps/googletest
 	url = https://github.com/google/googletest.git
+[submodule "deps/zlib"]
+	path = deps/zlib
+	url = https://github.com/madler/zlib.git
+	branch = develop
+	ignore = dirty

README.md

@@ -87,7 +87,7 @@ cmake --workflow --preset=release
 ## Dumping the Registry
 
 The emulator needs a registry dump to run, otherwise it will print `Bad hive file` errors.  
-You can create one by running the <a href="./src/grab-registry.bat">src/grab-registry.bat</a> script as administrator.  
+You can create one by running the <a href="./src/tools/grab-registry.bat">src/tools/grab-registry.bat</a> script as administrator.  
 This will create a `registry` folder that needs to be placed in the working directory of the emulator.
 
 ## Running Tests

deps/CMakeLists.txt

@@ -13,3 +13,4 @@ target_include_directories(reflect INTERFACE
 
 include(mini-gdbstub.cmake)
 include(googletest.cmake)
+include(zlib.cmake)

deps/zlib.cmake

@@ -0,0 +1,4 @@
+set(ZLIB_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+add_subdirectory(zlib)
+target_compile_definitions(zlibstatic PUBLIC ZLIB_CONST=1)
+target_include_directories(zlibstatic PUBLIC ${zlib_SOURCE_DIR} ${zlib_BINARY_DIR})

src/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_subdirectory(common)
+add_subdirectory(tools)
 add_subdirectory(emulator)
 add_subdirectory(unicorn-emulator)
 add_subdirectory(windows-emulator)

src/common/CMakeLists.txt

@@ -13,4 +13,5 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 target_link_libraries(emulator-common PUBLIC
   Threads::Threads
+  zlibstatic
 )

src/common/utils/compression.cpp

@@ -0,0 +1,105 @@
+#include "compression.hpp"
+
+#include <zlib.h>
+#include <cstring>
+
+namespace utils::compression
+{
+    namespace zlib
+    {
+        namespace
+        {
+            class zlib_stream
+            {
+              public:
+                zlib_stream()
+                {
+                    memset(&stream_, 0, sizeof(stream_));
+                    valid_ = inflateInit(&stream_) == Z_OK;
+                }
+
+                zlib_stream(zlib_stream&&) = delete;
+                zlib_stream(const zlib_stream&) = delete;
+                zlib_stream& operator=(zlib_stream&&) = delete;
+                zlib_stream& operator=(const zlib_stream&) = delete;
+
+                ~zlib_stream()
+                {
+                    if (valid_)
+                    {
+                        inflateEnd(&stream_);
+                    }
+                }
+
+                z_stream& get()
+                {
+                    return stream_; //
+                }
+
+                bool is_valid() const
+                {
+                    return valid_;
+                }
+
+              private:
+                bool valid_{false};
+                z_stream stream_{};
+            };
+        }
+
+        std::vector<std::uint8_t> decompress(const std::vector<std::uint8_t>& data)
+        {
+            std::vector<std::uint8_t> buffer{};
+            zlib_stream stream_container{};
+            if (!stream_container.is_valid())
+            {
+                return {};
+            }
+
+            int ret{};
+            size_t offset = 0;
+            static thread_local uint8_t dest[CHUNK] = {0};
+            auto& stream = stream_container.get();
+
+            do
+            {
+                const auto input_size = std::min(sizeof(dest), data.size() - offset);
+                stream.avail_in = static_cast<uInt>(input_size);
+                stream.next_in = reinterpret_cast<const Bytef*>(data.data()) + offset;
+                offset += stream.avail_in;
+
+                do
+                {
+                    stream.avail_out = sizeof(dest);
+                    stream.next_out = dest;
+
+                    ret = inflate(&stream, Z_NO_FLUSH);
+                    if (ret != Z_OK && ret != Z_STREAM_END)
+                    {
+                        return {};
+                    }
+
+                    buffer.insert(buffer.end(), dest, dest + sizeof(dest) - stream.avail_out);
+                } while (stream.avail_out == 0);
+            } while (ret != Z_STREAM_END);
+
+            return buffer;
+        }
+
+        std::vector<std::uint8_t> compress(const std::vector<std::uint8_t>& data)
+        {
+            std::vector<std::uint8_t> result{};
+            auto length = compressBound(static_cast<uLong>(data.size()));
+            result.resize(length);
+
+            if (compress2(reinterpret_cast<Bytef*>(result.data()), &length, reinterpret_cast<const Bytef*>(data.data()),
+                          static_cast<uLong>(data.size()), Z_BEST_COMPRESSION) != Z_OK)
+            {
+                return {};
+            }
+
+            result.resize(length);
+            return result;
+        }
+    }
+}

src/common/utils/compression.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <cstdint>
+
+#define CHUNK 16384u
+
+namespace utils::compression
+{
+    namespace zlib
+    {
+        std::vector<std::uint8_t> compress(const std::vector<std::uint8_t>& data);
+        std::vector<std::uint8_t> decompress(const std::vector<std::uint8_t>& data);
+    }
+};

src/tools/CMakeLists.txt

@@ -0,0 +1,4 @@
+if(WIN32)
+    add_subdirectory(dump-apiset)
+endif()
+

src/tools/dump-apiset/CMakeLists.txt

@@ -0,0 +1,17 @@
+file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS
+  *.cpp
+  *.hpp
+  *.rc
+)
+
+list(SORT SRC_FILES)
+
+add_executable(dump-apiset ${SRC_FILES})
+
+momo_assign_source_group(${SRC_FILES})
+
+target_link_libraries(dump-apiset PRIVATE
+  emulator-common
+)
+
+momo_strip_target(dump-apiset)

src/tools/dump-apiset/dump-apiset.cpp

@@ -0,0 +1,118 @@
+#include <cstdio>
+#include "platform/platform.hpp"
+#include "utils/compression.hpp"
+#include "utils/io.hpp"
+#include <vector>
+#include <iostream>
+
+void print_apiset(PAPI_SET_NAMESPACE apiSetMap);
+void create_header_file(const std::vector<uint8_t>& data);
+
+__forceinline PVOID GetCurrentProcessPeb()
+{
+#ifdef _WIN64
+    return (PVOID)__readgsqword(0x60);
+#else
+    return (PVOID)__readfsdword(0x30);
+#endif
+}
+
+int main()
+{
+    printf("Dump API-SET\n");
+    printf("------------\n\n");
+
+    const auto peb = (PPEB64)GetCurrentProcessPeb();
+    const auto apiSetMap = (PAPI_SET_NAMESPACE)(peb->ApiSetMap);
+
+    printf("APISET: 0x%p\n", apiSetMap);
+    printf("Version: %d\n", apiSetMap->Version);
+    printf("Size: %08X\n", apiSetMap->Size);
+    printf("Flags: %08X\n", apiSetMap->Flags);
+    printf("Count: %d\n", apiSetMap->Count);
+    printf("EntryOffset: %08X\n", apiSetMap->EntryOffset);
+    printf("HashOffset: %08X\n", apiSetMap->HashOffset);
+    printf("HashFactor: %08X\n", apiSetMap->HashFactor);
+    // print_apiset(apiSetMap);
+
+    // Compress the API-SET binary blob
+    const auto* dataPtr = reinterpret_cast<const uint8_t*>(apiSetMap);
+    std::vector<uint8_t> buffer(dataPtr, dataPtr + apiSetMap->Size);
+    auto compressed = utils::compression::zlib::compress(buffer);
+    if (compressed.empty())
+    {
+        printf("Failed to compress API-SET\n");
+        return 1;
+    }
+
+    // Dump the API-SET binary blob to disk
+    utils::io::write_file("api-set.bin", compressed, false);
+    printf("\nWrote API-SET to api-set.bin\n");
+    // create_header_file(compressed);
+
+    return 0;
+}
+
+void print_apiset(PAPI_SET_NAMESPACE apiSetMap)
+{
+    for (ULONG i = 0; i < apiSetMap->Count; i++)
+    {
+        auto entry = (PAPI_SET_NAMESPACE_ENTRY)((ULONG_PTR)apiSetMap + apiSetMap->EntryOffset +
+                                                i * sizeof(API_SET_NAMESPACE_ENTRY));
+        // printf("  Flags: %08X\n", entry->Flags);
+        // printf("  NameOffset: %08X\n", entry->NameOffset);
+        // printf("  NameLength: %08X\n", entry->NameLength);
+        // printf("  HashedLength: %08X\n", entry->HashedLength);
+        // printf("  ValueOffset: %08X\n", entry->ValueOffset);
+        // printf("  ValueCount: %08X\n", entry->ValueCount);
+
+        std::wstring name((wchar_t*)((ULONG_PTR)apiSetMap + entry->NameOffset), entry->NameLength / sizeof(wchar_t));
+        printf("-----------\n[%05d]: Contract Name: %ls\n", i, name.data());
+
+        for (ULONG x = 0; x < entry->ValueCount; x++)
+        {
+            auto value =
+                (PAPI_SET_VALUE_ENTRY)((ULONG_PTR)apiSetMap + entry->ValueOffset + x * sizeof(API_SET_VALUE_ENTRY));
+            // printf("  Value %d\n", x);
+            // printf("    Flags: %08X\n", value->Flags);
+            // printf("    NameOffset: %08X\n", value->NameOffset);
+            // printf("    NameLength: %08X\n", value->NameLength);
+            // printf("    ValueOffset: %08X\n", value->ValueOffset);
+            // printf("    ValueLength: %08X\n", value->ValueLength);
+
+            std::wstring hostName((wchar_t*)((ULONG_PTR)apiSetMap + value->NameOffset),
+                                  value->NameLength / sizeof(wchar_t));
+            std::wstring altName((wchar_t*)((ULONG_PTR)apiSetMap + value->ValueOffset),
+                                 value->ValueLength / sizeof(wchar_t));
+            printf("    HostName: %ls - AltName: %ls\n", hostName.empty() ? L"<none>" : hostName.data(),
+                   altName.empty() ? L"<none>" : altName.data());
+        }
+    }
+}
+
+// Internal
+void create_header_file(const std::vector<uint8_t>& data)
+{
+    FILE* output;
+    fopen_s(&output, "api-set.h", "w");
+    if (!output)
+    {
+        printf("Failed to create output file\n");
+        return;
+    }
+
+    fprintf(output, "#pragma once\n\n");
+    fprintf(output, "#include <stdint.h>\n\n");
+    fprintf(output, "const uint8_t api_set_blob[] = {\n");
+    for (ULONG i = 0; i < data.size(); i++)
+    {
+        fprintf(output, "0x%02X, ", data[i]);
+        if (i % 16 == 15)
+        {
+            fprintf(output, "\n");
+        }
+    }
+
+    fprintf(output, "};\n");
+    fclose(output);
+}

src/windows-emulator/windows_emulator.cpp

@@ -6,6 +6,10 @@
 
 #include <unicorn_x64_emulator.hpp>
 #include <utils/finally.hpp>
+#include "utils/compression.hpp"
+#include "utils/io.hpp"
+
+#include "apiset.hpp"
 
 constexpr auto MAX_INSTRUCTIONS_PER_TIME_SLICE = 100000;
 
@@ -133,15 +137,54 @@ namespace
         return api_set_map_obj;
     }
 
-    emulator_object<API_SET_NAMESPACE> build_api_set_map(x64_emulator& emu, emulator_allocator& allocator)
+    std::vector<uint8_t> decompress_apiset(const std::vector<uint8_t>& apiset)
     {
-        // TODO: fix
+        auto buffer = utils::compression::zlib::decompress(apiset);
+        if (buffer.empty())
+            throw std::runtime_error("Failed to decompress API-SET");
+        return buffer;
+    }
+
+    std::vector<uint8_t> obtain_api_set(apiset_location location)
+    {
+        switch (location)
+        {
 #ifdef OS_WINDOWS
-        const auto& orig_api_set_map = *NtCurrentTeb64()->ProcessEnvironmentBlock->ApiSetMap;
-        return clone_api_set_map(emu, allocator, orig_api_set_map);
+        case apiset_location::host: {
+            auto apiSetMap =
+                reinterpret_cast<const API_SET_NAMESPACE*>(NtCurrentTeb64()->ProcessEnvironmentBlock->ApiSetMap);
+            const auto* dataPtr = reinterpret_cast<const uint8_t*>(apiSetMap);
+            std::vector<uint8_t> buffer(dataPtr, dataPtr + apiSetMap->Size);
+            return buffer;
+        }
 #else
-        return clone_api_set_map(emu, allocator, {});
+        case apiset_location::host:
+            throw std::runtime_error("The APISET host location is not supported on this platform");
 #endif
+        case apiset_location::file: {
+            auto apiset = utils::io::read_file("api-set.bin");
+            if (apiset.empty())
+                throw std::runtime_error("Failed to read file api-set.bin");
+            return decompress_apiset(apiset);
+        }
+        case apiset_location::default_windows_10: {
+            const std::vector<uint8_t> apiset{apiset_w10, apiset_w10 + sizeof(apiset_w10)};
+            return decompress_apiset(apiset);
+        }
+        case apiset_location::default_windows_11: {
+            const std::vector<uint8_t> apiset{apiset_w11, apiset_w11 + sizeof(apiset_w11)};
+            return decompress_apiset(apiset);
+        }
+        default:
+            throw std::runtime_error("Bad API set location");
+        }
+    }
+
+    emulator_object<API_SET_NAMESPACE> build_api_set_map(x64_emulator& emu, emulator_allocator& allocator,
+                                                         apiset_location location = apiset_location::host)
+    {
+        return clone_api_set_map(emu, allocator,
+                                 reinterpret_cast<const API_SET_NAMESPACE&>(*obtain_api_set(location).data()));
     }
 
     emulator_allocator create_allocator(emulator& emu, const size_t size)
@@ -247,10 +290,17 @@ namespace
             proc_params.MaximumLength = proc_params.Length;
         });
 
+// TODO: make this configurable
+#ifdef OS_WINDOWS
+        apiset_location apiset_loc = apiset_location::host;
+#else
+        apiset_location apiset_loc = apiset_location::default_windows_11;
+#endif
+
         context.peb.access([&](PEB64& peb) {
             peb.ImageBaseAddress = nullptr;
             peb.ProcessParameters = context.process_params.ptr();
-            peb.ApiSetMap = build_api_set_map(emu, allocator).ptr();
+            peb.ApiSetMap = build_api_set_map(emu, allocator, apiset_loc).ptr();
 
             peb.ProcessHeap = nullptr;
             peb.ProcessHeaps = nullptr;

src/windows-emulator/windows_emulator.hpp

@@ -22,6 +22,14 @@ struct emulator_settings
     bool use_relative_time{false};
 };
 
+enum class apiset_location
+{
+    host,
+    file,
+    default_windows_10,
+    default_windows_11
+};
+
 class windows_emulator
 {
   public: