Fix NtCreateFile's create_disposition behavior and other fixes (#700)

This PR does the following:
- [Fix NtCreateFile's create_disposition
behavior](ffd7058bec).
Previously, there were cases being mishandled by the emulator, this
change should address all of them.
- [Fix wrong return value of
user_handle_table::get_server_info](fc036ca2c5).
- [Fix inadvertent stack pointer overwriting in callback
hook](f5fe795028).
This was an issue because callback completion can dispatch another
callback, so the rsp after completion shouldn't be ignored.
- [Make Atom
case-insensitive](2c93915ecc).
This appears to match Windows behavior.
- [Add useful comment to user_callback_dispatch
header](271fb0f270).

src/common/platform/status.hpp

@@ -43,6 +43,7 @@ using NTSTATUS = std::uint32_t;
 #define STATUS_ACCESS_DENIED              ((NTSTATUS)0xC0000022L)
 #define STATUS_BUFFER_TOO_SMALL           ((NTSTATUS)0xC0000023L)
 #define STATUS_OBJECT_NAME_NOT_FOUND      ((NTSTATUS)0xC0000034L)
+#define STATUS_OBJECT_NAME_COLLISION      ((NTSTATUS)0xC0000035L)
 #define STATUS_INVALID_PAGE_PROTECTION    ((NTSTATUS)0xC0000045L)
 #define STATUS_MUTANT_NOT_OWNED           ((NTSTATUS)0xC0000046L)
 #define STATUS_SEMAPHORE_LIMIT_EXCEEDED   ((NTSTATUS)0xC0000047L)

src/common/platform/user.hpp

@@ -23,7 +23,7 @@ struct USER_HANDLEENTRY
     uint64_t pHead;
     uint64_t pOwner;
     uint64_t unknown;
-    uint8_t bType;
+    EMULATOR_CAST(uint8_t, USER_HANDLETYPE) bType;
     uint8_t bFlags;
     uint16_t wUniq;
 };
@@ -31,10 +31,10 @@ static_assert(sizeof(USER_HANDLEENTRY) == 0x20);
 
 struct USER_SHAREDINFO
 {
-    uint64_t psi;
-    uint64_t aheList;
+    EMULATOR_CAST(uint64_t, USER_SERVERINFO*) psi;
+    EMULATOR_CAST(uint64_t, USER_HANDLEENTRY*) aheList;
     uint32_t HeEntrySize;
-    uint64_t pDispInfo;
+    EMULATOR_CAST(uint64_t, USER_DISPINFO*) pDispInfo;
     uint8_t unknown[0xFF];
 };
 

src/windows-emulator/process_context.cpp

@@ -503,7 +503,7 @@ void process_context::setup_callback_hook(windows_emulator& win_emu, memory_mana
             target_rip += 2;
         }
 
-        const uint64_t ret_stack_ptr = frame.rsp - sizeof(emulator_pointer);
+        const uint64_t ret_stack_ptr = emu.reg(x86_register::rsp) - sizeof(emulator_pointer);
         emu.write_memory(ret_stack_ptr, &target_rip, sizeof(target_rip));
         emu.reg(x86_register::rsp, ret_stack_ptr);
     });
@@ -657,7 +657,7 @@ std::optional<uint16_t> process_context::find_atom(const std::u16string_view nam
 {
     for (auto& entry : this->atoms)
     {
-        if (entry.second.name == name)
+        if (utils::string::equals_ignore_case(std::u16string_view{entry.second.name}, name))
         {
             ++entry.second.ref_count;
             return entry.first;
@@ -680,7 +680,7 @@ uint16_t process_context::add_or_find_atom(std::u16string name)
     std::optional<uint16_t> last_entry{};
     for (auto& entry : this->atoms)
     {
-        if (entry.second.name == name)
+        if (utils::string::equals_ignore_case(entry.second.name, name))
         {
             ++entry.second.ref_count;
             return entry.first;
@@ -714,7 +714,7 @@ bool process_context::delete_atom(const std::u16string& name)
 {
     for (auto it = atoms.begin(); it != atoms.end(); ++it)
     {
-        if (it->second.name == name)
+        if (utils::string::equals_ignore_case(it->second.name, name))
         {
             if (--it->second.ref_count == 0)
             {

src/windows-emulator/syscalls/file.cpp

@@ -1020,7 +1020,8 @@ namespace syscalls
             return STATUS_OBJECT_NAME_NOT_FOUND;
         }
 
-        const bool is_directory = std::filesystem::is_directory(c.win_emu.file_sys.translate(path), ec);
+        const auto host_path = c.win_emu.file_sys.translate(path);
+        const bool is_directory = std::filesystem::is_directory(host_path, ec);
 
         if (is_directory || create_options & FILE_DIRECTORY_FILE)
         {
@@ -1028,7 +1029,7 @@ namespace syscalls
 
             if (create_disposition & FILE_CREATE)
             {
-                create_directory(c.win_emu.file_sys.translate(path), ec);
+                create_directory(host_path, ec);
 
                 if (ec)
                 {
@@ -1048,6 +1049,27 @@ namespace syscalls
 
         c.win_emu.callbacks.on_generic_access("Opening file", f.name);
 
+        const bool file_exists = std::filesystem::exists(host_path, ec);
+
+        if (create_disposition == FILE_CREATE && file_exists)
+        {
+            return STATUS_OBJECT_NAME_COLLISION;
+        }
+
+        if ((create_disposition == FILE_OVERWRITE || create_disposition == FILE_OPEN) && !file_exists)
+        {
+            return STATUS_OBJECT_NAME_NOT_FOUND;
+        }
+
+        if (create_disposition == FILE_OPEN_IF && !file_exists)
+        {
+            std::ofstream touch(host_path, std::ios::binary | std::ios::app);
+            if (!touch)
+            {
+                return STATUS_ACCESS_DENIED;
+            }
+        }
+
         std::u16string mode = map_mode(desired_access, create_disposition);
 
         if (mode.empty() || path.is_relative())

src/windows-emulator/user_callback_dispatch.hpp

@@ -6,6 +6,7 @@
 //       In the real implementation, the kernel invokes ntdll!KiUserCallbackDispatcher and passes a callback
 //       index that refers to an entry in PEB->KernelCallbackTable. The dispatcher then looks up the function
 //       pointer in that table and invokes the corresponding user-mode callback.
+//       See Also: https://web.archive.org/web/20080717175308/http://www.nynaeve.net/?p=204
 
 template <typename... Args>
 void prepare_call_stack(x86_64_emulator& emu, uint64_t return_address, Args... args)

src/windows-emulator/user_handle_table.hpp

@@ -33,7 +33,7 @@ class user_handle_table
         handle_table_addr_ = this->allocate_memory(handle_table_size, memory_permission::read);
     }
 
-    emulator_object<USER_SHAREDINFO> get_server_info() const
+    emulator_object<USER_SERVERINFO> get_server_info() const
     {
         return {*memory_, server_info_addr_};
     }