I've followed your suggestions but CreateWindowEx() still fails. GetLastError() returns 0, which isn't very helpful. As to your question, I was referring to the integer that EVERY function has to return (as per the mIRC help file) so that mIRC knows whether we want to halt processing, return the data to a $dll call, continue processing, etc. My code is below (changed slightly as I don't use unicode now):

dll.h:

Code:
#ifndef _DLL_H_
#define _DLL_H_

#include <windows.h>

#define DLLIMPORT __declspec (dllexport) int __stdcall

#define MYMENU_EXIT         (WM_APP + 101)
#define MYMENU_MESSAGEBOX   (WM_APP + 102) 

void __stdcall LoadDll(LOADINFO*);
int __stdcall UnloadDll(int);
DLLIMPORT HelloWorld (HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause);
DLLIMPORT dllname (HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause);
DLLIMPORT window(HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause);

#endif /* _DLL_H_ */


And dllmain.c:

Code:
#include "dll.h"
#include <stdio.h>
#include <stdlib.h>

// WndProc for the new window
LRESULT CALLBACK DLLWindowProc (HWND, UINT, WPARAM, LPARAM);

static HINSTANCE dllInstance;
static HWND mIRC_window;
static WNDPROC mWndProc;
static HWND myHwnd;

typedef struct {
	DWORD  mVersion;
	HWND   mHwnd;
	BOOL   mKeep;
	BOOL   mUnicode;
} LOADINFO;

int __stdcall UnloadDll(int mTimeout) {
	if (mTimeout == 0) { /* user called /dll -u (or mIRC is shutting down) */
		/* Remove window hook */
		SetWindowLong(mIRC_window, GWL_WNDPROC, (LONG)mWndProc);
    }
	return 0; // keep the dll loaded
}

void __stdcall LoadDLL(LOADINFO *load) {
	load->mKeep = TRUE;
	// setup window subclass to hook the dll window event loop
	mIRC_window = load->mHwnd;
	mWndProc = (WNDPROC)SetWindowLong(mIRC_window, GWL_WNDPROC, (LONG)DLLWindowProc); 
}

// /dll mydll.dll HelloWorld works as expected
DLLIMPORT HelloWorld (HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause)
{
    MessageBox (0, "Hello World from DLL!\n", "Hi", MB_ICONINFORMATION);
    return 1;
}

// this one works too
DLLIMPORT dllname (HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause)
{
    char szFileName[MAX_PATH];
	GetModuleFileName((HMODULE)dllInstance, szFileName, MAX_PATH);
	MessageBox(aWnd, szFileName, "This DLL is:", MB_OK | MB_ICONINFORMATION);
	lstrcpyA(data, "yay");
	return 3;
}

// Create our window's Menu
HMENU CreateDLLWindowMenu()
{
	HMENU hMenu;
	hMenu = CreateMenu();
	HMENU hMenuPopup;
        if(hMenu==NULL)
            return FALSE;
        hMenuPopup = CreatePopupMenu();
	AppendMenu (hMenuPopup, MF_STRING, MYMENU_EXIT, TEXT("Exit"));
        AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("File")); 

	hMenuPopup = CreatePopupMenu();
        AppendMenu (hMenuPopup, MF_STRING,MYMENU_MESSAGEBOX, TEXT("MessageBox")); 
        AppendMenu (hMenu, MF_POPUP, (UINT_PTR) hMenuPopup, TEXT("Test")); 
	return hMenu;
}

// this one fails at CreateWindowEx()
DLLIMPORT window(HWND mWnd, HWND aWnd, char *data, char *parms, BOOL show, BOOL nopause) {
	HMENU hMenu = CreateDLLWindowMenu();
	WNDCLASSEX wc;
        wc.hInstance = dllInstance;
	wc.lpszClassName = "DLLWindowClass";
        wc.lpfnWndProc = DLLWindowProc;
        wc.style = CS_DBLCLKS;
        wc.cbSize = sizeof (WNDCLASSEX);
        wc.hIcon = LoadIcon (NULL, IDI_APPLICATION);
        wc.hIconSm = LoadIcon (NULL, IDI_APPLICATION);
        wc.hCursor = LoadCursor (NULL, IDC_ARROW);
        wc.lpszMenuName = NULL;
        wc.cbClsExtra = 0;
        wc.cbWndExtra = 0;
        wc.hbrBackground = (HBRUSH) COLOR_BACKGROUND;
	if (!RegisterClassEx (&wc)) {
		MessageBox(mIRC_window, "Failed to register window class!", "Info", MB_OK | MB_ICONINFORMATION);
		//return 1;
	}

	myHwnd = CreateWindowEx (0, "DLLWindowClass", "Window Title", WS_EX_PALETTEWINDOW, CW_USEDEFAULT,
		CW_USEDEFAULT, 400, 300, mIRC_window, hMenu, dllInstance, NULL );
	if (myHwnd == NULL) {
		DWORD Error = GetLastError();
		char ErrorBuffer[ 1024 ];
		wsprintf( ErrorBuffer, "Error creating window. Error code, decimal %d, hexadecimal %X.", Error, Error );
		MessageBox(mIRC_window, ErrorBuffer, "Error", MB_ICONHAND );
		MessageBox(mIRC_window, "Failed to create window!", "Info", MB_OK | MB_ICONINFORMATION);
		//return 1;
	}
	UpdateWindow(myHwnd); // possibly unnecessary
	ShowWindow (myHwnd, SW_SHOWNORMAL);
	lstrcpyA(data, "meh");
	return 3;
}

LRESULT CALLBACK DLLWindowProc (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
	if (hwnd == myHwnd) {
	    switch (message) {
	        case WM_COMMAND:
	            switch(wParam) {
		                       case MYMENU_EXIT:
                         		    SendMessage(hwnd, WM_CLOSE, 0, 0);
                                    break;
                               case MYMENU_MESSAGEBOX:
		                       	    MessageBox(hwnd, "Test", "MessageBox",MB_OK);
                                    break;
                               }
                break;
		    case WM_DESTROY:
		         PostQuitMessage (0);
		         break;
	        default:
		            return DefWindowProc (hwnd, message, wParam, lParam);
	    }
	    // return 0;
	} 

	/* Send the rest to mIRC */
	else { return CallWindowProc(mWndProc, hwnd, message, wParam, lParam); }
}

BOOL APIENTRY DllMain (HINSTANCE hInst     /* Library instance handle. */ ,
                       DWORD reason        /* Reason this function is being called. */ ,
                       LPVOID reserved     /* Not used. */ )
{
    dllInstance = hInst;
    switch (reason)
    {
      case DLL_PROCESS_ATTACH:
        break;

      case DLL_PROCESS_DETACH:
        break;

      case DLL_THREAD_ATTACH:
        break;

      case DLL_THREAD_DETACH:
        break;
    }

    /* Returns TRUE on success, FALSE on failure */
    return TRUE;
}