Getting Invalid Hook Procedure error after injected dll in windows hook

Question

I am trying to hook into windows globally by injecting a dll. My code works with python2.7, but sadly fails, when using Python3.7 with the already mentioned error. Here is an example code:

user32.SetWindowsHookExA.errcheck = errcheck_bool
user32.SetWindowsHookExA.restype = HHOOK
user32.SetWindowsHookExA.argtypes = (c_int,     # _In_ idHook
                                     HOOKPROC,  # _In_ lpfn
                                     HINSTANCE, # _In_ hMod
                                     DWORD)     # _In_ dwThreadId

user32.CallNextHookEx.restype = LRESULT
user32.CallNextHookEx.argtypes = (HHOOK,  # _In_opt_ hhk
                                  c_int,  # _In_     nCode
                                  WPARAM, # _In_     wParam
                                  LPARAM) # _In_     lParam

user32.GetMessageW.argtypes = (LPMSG, # _Out_    lpMsg
                               HWND,  # _In_opt_ hWnd
                               UINT,  # _In_     wMsgFilterMin
                               UINT)  # _In_     wMsgFilterMax

user32.TranslateMessage.argtypes = (LPMSG,)
user32.DispatchMessageW.argtypes = (LPMSG,)

GetModuleHandle = ctypes.windll.kernel32.GetModuleHandleA
GetModuleHandle.restype = POINTER(c_void_p)

LoadLibrary = ctypes.windll.kernel32.LoadLibraryA
LoadLibrary.restype = HINSTANCE

GetProcAddress = ctypes.windll.kernel32.GetProcAddress
GetProcAddress.restype = HOOKPROC

user32.GetWindowThreadProcessId.restype = DWORD

def pointer_msg_loop():
    lib = LoadLibrary(r'C:\Users\Braun\Documents\BA_Thesis\ba-oliver-braun-logging-tool-code\MessagesDll\x64\Release\Dll.dll')
    handle = GetModuleHandle(r'C:\Users\Braun\Documents\BA_Thesis\ba-oliver-braun-logging-tool-code\MessagesDll\x64\Release\Dll.dll')
    print(lib)
    print(handle)
    procedure = GetProcAddress(handle, "meconnect".encode())
    print(procedure)
    print('correct value procedure')
    tHook = user32.SetWindowsHookExA(WH_GETMESSAGE, procedure, lib, 0)
    time.sleep(30)
    user32.UnhookWindowsHookEx(tHook)
    print(tHook)
    msg = MSG()
    while True:
        bRet = user32.GetMessageW(byref(msg), None, 0, 0)
        if not bRet:
            break
        if bRet == -1:
            raise WinError(get_last_error())
        user32.TranslateMessage(byref(msg))
        user32.DispatchMessageW(byref(msg))

if __name__ == '__main__':
    import time
    import datetime
    import threading
    control_key_pressed = False
    startTime = datetime.datetime.now()
    tmouse = threading.Thread(target=mouse_msg_loop)
    tkeyboard = threading.Thread(target=keyboard_msg_loop)
    ttouch = threading.Thread(target=pointer_msg_loop)
    tmouse.start()
    tkeyboard.start()
    ttouch.start()
    while True:
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            user32.PostThreadMessageW(tmouse.ident, WM_QUIT, 0, 0)
            break

DLL:

// dllmain.cpp : Defines the entry point for the DLL application.
#include "pch.h"
#pragma data_seg("Shared")
#pragma data_seg()
#pragma comment(linker,"/section:Shared,rws")

#include <windows.h>
#include <stdio.h>
#include <string.h>

HHOOK tHook;

extern "C" __declspec(dllexport) LRESULT meconnect(int code, WPARAM wParam, LPARAM lParam) {
    BOOL EnableMouseInPointer = TRUE;
    if (code == HC_ACTION) {
        LPMSG data = (LPMSG)lParam;
        if (data->message == WM_POINTERDOWN) {
            MessageBoxA(NULL, "eee", NULL, 0);
        }
    }
    return(CallNextHookEx(tHook, code, wParam, lParam));
}

Some interesting observations: When changing LoadLibraryA to LoadLibraryW even python2.7 fails. This seems really obvious and i suspect its because of maybe the different string types that both versions enforce? But yeah what can i change to make it work in Python3.7

Answer 1

In Python 3.7, the result of sys.getdefaultencoding() is 'utf-8'. This answer from @jfs metioned that

sys.getdefaultencoding() always runs 'ascii' on all systems of Python 2, unless you rewrite it.

This may be why you got it wrong.

1. use b'' instead of r'', or just use Wide-byte function.
2. GetProcAddress has no Wide-byte version. Just use GetProcAddress(handle, b'meconnect') as well.
3. tHook in DLL is not defined, and it will cause undefined behavior in next hook.
4. You'd better put the message loop before UnhookWindowsHookEx and instead of the sleep method.

You can call SetWindowsHookEx in the DLL. Here is the sample. DLL:

// dllmain.cpp : Defines the entry point for the DLL application.
#include "pch.h"
#include <windows.h>
#include <stdio.h>
#include <string.h>
HHOOK tHook;
HMODULE hinstDLL;
LRESULT CALLBACK meconnect(int code, WPARAM wParam, LPARAM lParam) {
    BOOL EnableMouseInPointer = TRUE;
    if (code == HC_ACTION) {
        LPMSG data = (LPMSG)lParam;
        if (data->message == WM_POINTERDOWN) {
            MessageBoxA(NULL, "eee", NULL, 0);
        }
    }
    return(CallNextHookEx(tHook, code, wParam, lParam));
}
extern "C" __declspec(dllexport) BOOL SetHook()
{
    tHook = SetWindowsHookEx(WH_GETMESSAGE, meconnect, hinstDLL, 0);

    if (tHook == NULL)
        return FALSE;
    else
        return TRUE;
}
extern "C" __declspec(dllexport) BOOL UnHook()
{
    return UnhookWindowsHookEx(tHook);
}


BOOL APIENTRY DllMain( HMODULE hModule,
                       DWORD  ul_reason_for_call,
                       LPVOID lpReserved
                     )
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        hinstDLL = hModule;
        break;
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

python:

import threading
import ctypes
from ctypes import wintypes
from ctypes import *
import win32api
import win32gui
def pointer_msg_loop():
    lib = cdll.LoadLibrary(r'PATH\DLL.dll')
    print(lib)
    res = lib.SetHook()
    print(res)
    win32gui.PumpMessages()
    res = lib.UnHook()


if __name__ == '__main__':
    ttouch = threading.Thread(target=pointer_msg_loop)
    ttouch.start()
    ttouch.join()