OK, so I've hooked into the mIRC WndProc, but now CreateWindowEx seems to always fail (message box informs me so, see code below).

Code:
#include <windows.h>

#define MYMENU_EXIT         (WM_APP + 101)
#define MYMENU_MESSAGEBOX   (WM_APP + 102) 
#define MY_MSGLOOP	    (WM_USER+900)

// WndProc for the new window
LRESULT CALLBACK DLLWindowProc (HWND, UINT, WPARAM, LPARAM);

static HINSTANCE dllInstance;
static HWND mIRC_window;
static WNDPROC mWndProc;

typedef struct {
	DWORD  mVersion;
	HWND   mHwnd;
	BOOL   mKeep;
	BOOL   mUnicode;
} LOADINFO;

void __stdcall LoadDll(LOADINFO*);
int __stdcall UnloadDll(int);

int __stdcall UnloadDll(int mTimeout) {
	if (mTimeout == 0) { /* user called /dll -u (or mIRC is shutting down) */
		/* Remove window hook */
		SetWindowLong(mIRC_window, GWL_WNDPROC, (LONG)mWndProc);
        }
	return 0; // keep the dll loaded
}

void __stdcall LoadDLL(LOADINFO *load) {
	load->mKeep = TRUE;
	// setup window subclass to hook the dll window event loop
	mIRC_window = load->mHwnd;
	mWndProc = (WNDPROC)SetWindowLong(mIRC_window, GWL_WNDPROC, (LONG)DLLWindowProc); 
}

// Create our window's Menu
HMENU CreateDLLWindowMenu()
{
	HMENU hMenu;
	hMenu = CreateMenu();
	HMENU hMenuPopup;
        if(hMenu==NULL)
            return FALSE;
        hMenuPopup = CreatePopupMenu();
	AppendMenu (hMenuPopup, MF_STRING, MYMENU_EXIT, TEXT("Exit"));
        AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("File")); 

	hMenuPopup = CreatePopupMenu();
        AppendMenu (hMenuPopup, MF_STRING,MYMENU_MESSAGEBOX, TEXT("MessageBox")); 
        AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("Test")); 
	return hMenu;
}

// Our new window's proc
LRESULT CALLBACK DLLWindowProc (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
	if (message == MY_MSGLOOP) {
	    switch (message) {
	        case WM_COMMAND:
	            switch(wParam) {
		        case MYMENU_EXIT:
			    SendMessage(hwnd, WM_CLOSE, 0, 0);
                            break;
                        case MYMENU_MESSAGEBOX:
			    MessageBox(hwnd, L"Test", L"MessageBox",MB_OK);
                            break;
                    }
                    break;
		case WM_DESTROY:
		    PostQuitMessage (0);
		    break;
	        default:
		    return DefWindowProc (hwnd, message, wParam, lParam);
	    }
	    return 0;
	} 

	/* Send the rest to mIRC */
	return CallWindowProc(mWndProc, hwnd, message, wParam, lParam);
}

BOOL APIENTRY DllMain(HINSTANCE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
	dllInstance = hModule;
        return TRUE;
}

int __stdcall dllname(HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause) {
	wchar_t szFileName[MAX_PATH];
	GetModuleFileName((HMODULE)dllInstance, szFileName, MAX_PATH);
	MessageBox(aWnd, szFileName, L"This DLL is:", MB_OK | MB_ICONINFORMATION);
	lstrcpyA(data, "yay");
	return 3;
}

int __stdcall window(HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause) {
	HMENU hMenu = CreateDLLWindowMenu();
	WNDCLASSEX wc;
        wc.hInstance = dllInstance;
	wc.lpszClassName = (LPCWSTR)L"DLLWindowClass";
        wc.lpfnWndProc = DLLWindowProc;
        wc.style = CS_DBLCLKS;
        wc.cbSize = sizeof (WNDCLASSEX);
        wc.hIcon = LoadIcon (NULL, IDI_APPLICATION);
        wc.hIconSm = LoadIcon (NULL, IDI_APPLICATION);
        wc.hCursor = LoadCursor (NULL, IDC_ARROW);
        wc.lpszMenuName = NULL;
        wc.cbClsExtra = 0;
        wc.cbWndExtra = 0;
        wc.hbrBackground = (HBRUSH) COLOR_BACKGROUND;
	if (!RegisterClassEx (&wc)) {
		MessageBox(mIRC_window, L"Failed to register window class!", L"Info", MB_OK | MB_ICONINFORMATION);
		//return 1;
	}
	HWND hwnd = CreateWindowEx (0, L"DLLWindowClass", L"Window Title", WS_EX_PALETTEWINDOW, CW_USEDEFAULT,
		CW_USEDEFAULT, 400, 300, mIRC_window, hMenu, dllInstance, NULL );
	if (!hwnd) {
		MessageBox(mIRC_window, L"Failed to create window!", L"Info", MB_OK | MB_ICONINFORMATION);
		//return 1;
	}
	UpdateWindow(hwnd); // possibly unnecessary
	ShowWindow (hwnd, SW_SHOWNORMAL);
	/* the while loop below would obviously block, but this function has to return
	   to let mirc know what we want to do, so how do I dispatch messages to my
	   message loop without creating a different thread? (code commented for now)
	while (1) {
		PostMessage(mIRC_window, MY_MSGLOOP, 0, 0);
		Sleep(1);
	} */
	lstrcpyA(data, "meh");
	return 3;
}

Any help is greatly appreciated. Also, 7ramy, I couldn't compile your code. What language is this written in? The Func(start) at the bottom doesn't look like C (or C++).

Last edited by deadbeef; 14/01/11 12:13 AM.