The issue arises because a window/dialog procedure is an alias(typedef) for a pointer to a function taking four parameters and returning a non-void 'value'. Your non-static member class function does not match this type(it's encumbered with this), while a static function does. Obviously, you could also make the callback procedure global or better, if following this approach, limit its scope within a suitable namespace.
This is a common problem which is typically solved by making the callback member function static and providing a mechanism whereby a pointer to a specific class object is available to that static function to forward or otherwise handle the messages. There are other approaches such as using hooks or assembly thunks but they are much more complex in both terms of implementation and comprehension.
Consider the following code using a static dialog procedure; the process is analagous for windows but with CreatWindowEx's final parameter used to pass this with WM_NCREATE initially intercepted instead of WM_INITDIALOG:
//dlg.cpp
#include <windows.h>
#include <tchar.h>
#include "resources.h"
class Dialog
{
private:
HWND hDlg;
protected:
public:
Dialog():hDlg(0){};
virtual ~Dialog(){};
INT_PTR Create(const int id,const HWND hParent = 0)
{
return DialogBoxParam(GetModuleHandle(0),MAKEINTRESOURCE(id),hParent,DlgProc,
reinterpret_cast<LPARAM>(this));
}
static INT_PTR CALLBACK DlgProc(HWND hwnd,UINT uMsg,WPARAM wp,LPARAM lp)
{
Dialog *pDlg=reinterpret_cast<Dialog*>(GetWindowLongPtr(hwnd,GWLP_USERDATA));
if (!pDlg)
{
if (uMsg == WM_INITDIALOG)
{
pDlg = reinterpret_cast<Dialog*>(lp);
pDlg->hDlg = hwnd;
SetWindowLongPtr(hwnd,GWLP_USERDATA,lp);
}
else
{
return 0; //let system deal with message
}
}
//forward message to member function handler
return pDlg->ClassDlgProc(uMsg,wp,lp);
}
INT_PTR ClassDlgProc(UINT uMsg,WPARAM wp,LPARAM lp)
{
//handle messages normally
if (uMsg == WM_COMMAND)
{
if (LOWORD(wp) == IDOK || LOWORD(wp) == IDCANCEL)
{
EndDialog(hDlg,LOWORD(wp));
return 1;
}
}
return 0;
}
};
int WINAPI WinMain(HINSTANCE,HINSTANCE,LPSTR,int)
{
Dialog my_dlg;
INT_PTR ip = my_dlg.Create(IDD_DLG);
return 0;
} //script.rc
#if !defined __BORLANDC__
#include <windows.h>
#endif
#include "resources.h"
IDD_DLG DIALOGEX 0, 0, 252, 149
STYLE DS_CENTER | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_POPUP | WS_VISIBLE |
WS_CAPTION | WS_SYSMENU | WS_THICKFRAME
CAPTION "Dialog Box: Modal"
FONT 8, "MS Sans Serif"
{
DEFPUSHBUTTON "OK",IDOK,22,62,50,14
PUSHBUTTON "Cancel",IDCANCEL,111,63,50,14
} //resources.h
#define IDD_DLG 200