failed attempt to use a hook (WH_CALLWNDPROC) instead of a window procedure.

This works well until we try to SendMessage from our proxy window to Firefox
windows. SendMessage crashes when sending to a Firefox window with our hook
registered, or to *any* window (tested on our proxy window and arbitrary
launched programs).

src/modules/ctypes/winnt/kernel32.jsm

@@ -33,6 +33,8 @@ function kernel32_defines(lib) {
 
   lib.lazy_bind("LoadLibraryW", win32.HMODULE, win32.LPCTSTR);
   lib.lazy_bind("GetProcAddress", win32.FARPROC, win32.HMODULE, win32.LPCSTR);
+  lib.lazy_bind("GetCurrentThreadId", win32.DWORD);
+
 
 }
 

src/modules/ctypes/winnt/user32.jsm

@@ -134,6 +134,55 @@ function user32_defines(lib) {
   this.WS_OVERLAPPEDWINDOW = (this.WS_OVERLAPPED | this.WS_CAPTION | this.WS_SYSMENU | this.WS_THICKFRAME | this.WS_MINIMIZEBOX | this.WS_MAXIMIZEBOX);
   this.WS_TILEDWINDOW      = (this.WS_OVERLAPPED | this.WS_CAPTION | this.WS_SYSMENU | this.WS_THICKFRAME | this.WS_MINIMIZEBOX | this.WS_MAXIMIZEBOX);
 
+  this.CWPSTRUCT = ctypes.StructType("CWPSTRUCT", [
+    { "lParam": win32.LPARAM },
+    { "wParam": win32.WPARAM },
+    { "message": win32.UINT },
+    { "hwnd": win32.HWND }
+  ]);
+
+  this.CWPRETSTRUCT = ctypes.StructType("CWPRETSTRUCT", [
+    { "lResult": win32.LRESULT },
+    { "lParam": win32.LPARAM },
+    { "wParam": win32.WPARAM },
+    { "message": win32.UINT },
+    { "hwnd": win32.HWND }
+  ]);
+
+  this.HOOKPROC = ctypes.FunctionType(
+    WinCbABI, win32.LRESULT,
+    [ctypes.int, win32.WPARAM, win32.LPARAM]).ptr;
+
+  lib.lazy_bind("SetWindowsHookExW", win32.HHOOK, ctypes.int, this.HOOKPROC, win32.HINSTANCE, win32.DWORD);
+  lib.lazy_bind("CallNextHookEx", win32.LRESULT, win32.HHOOK, ctypes.int, win32.WPARAM, win32.LPARAM);
+  lib.lazy_bind("UnhookWindowsHookEx", win32.BOOL, win32.HHOOK);
+
+  this.WH_MIN             = (-1);
+  this.WH_MSGFILTER       = (-1);
+  this.WH_JOURNALRECORD   = 0;
+  this.WH_JOURNALPLAYBACK = 1;
+  this.WH_KEYBOARD        = 2;
+  this.WH_GETMESSAGE      = 3;
+  this.WH_CALLWNDPROC     = 4;
+  this.WH_CBT             = 5;
+  this.WH_SYSMSGFILTER    = 6;
+  this.WH_MOUSE           = 7;
+  this.WH_HARDWARE        = 8;
+  this.WH_DEBUG           = 9;
+  this.WH_SHELL           = 10;
+  this.WH_FOREGROUNDIDLE  = 11;
+  this.WH_CALLWNDPROCRET  = 12;
+  this.WH_KEYBOARD_LL     = 13;
+  this.WH_MOUSE_LL        = 14;
+
+  this.HC_ACTION      = 0;
+  this.HC_GETNEXT     = 1;
+  this.HC_SKIP        = 2;
+  this.HC_NOREMOVE    = 3;
+  this.HC_NOREM       = this.HC_NOREMOVE;
+  this.HC_SYSMODALON  = 4;
+  this.HC_SYSMODALOFF = 5;
+
 }
 
 new ctypes_library(USER32_LIBNAME, USER32_ABIS, user32_defines, this);

src/modules/ctypes/winnt/win32.jsm

@@ -39,6 +39,7 @@ var win32 = new function() {
   this.HMENU     = this.HANDLE;
   this.HBRUSH    = this.HICON;
   this.HCURSOR   = this.HANDLE;
+  this.HHOOK     = this.HANDLE;
   this.TCHAR     = ctypes.jschar, // Mozilla compiled with UNICODE/_UNICODE macros and wchar_t = jschar
   this.LPSTR     = ctypes.char.ptr;
   this.LPCSTR    = ctypes.char.ptr;

src/modules/winnt/FiretrayStatusIcon.jsm

@@ -124,11 +124,15 @@ firetray.StatusIcon = {
     if (uMsg === firetray.Win32.WM_TASKBARCREATED) {
       log.info("____________TASKBARCREATED");
 
+    } else if (uMsg === firetray.Win32.WM_TRAYMESSAGEFWD) {
+      log.debug("ProxyWindowProc WM_TRAYMESSAGEFWD reached!");
+
     } else if (uMsg === firetray.Win32.WM_TRAYMESSAGE) {
 
       switch (+lParam) {
       case win32.WM_LBUTTONUP:
         log.debug("WM_LBUTTONUP");
+        let rv = user32.SendMessageW(hWnd, firetray.Win32.WM_TRAYMESSAGEFWD, 0, 1);
         break;
       case win32.WM_RBUTTONUP:
         log.debug("WM_RBUTTONUP");
@@ -142,27 +146,30 @@ firetray.StatusIcon = {
       default:
       }
 
-    }
+try {
 
-/*
-    // CallWindowProcW() on a non-moz window works fine
-    let procPrev = firetray.StatusIcon.callbacks.procPrev;
-    log.debug("  procPrev="+procPrev);
-    let rv = user32.CallWindowProcW(procPrev, hWnd, uMsg, wParam, lParam);
-    log.debug("  CallWindowProc="+rv);
-    return rv;
-*/
+      for (let wid in firetray.Handler.windows) {
+        let hwnd = firetray.Win32.hexStrToHwnd(wid);
+        let rv = user32.SendMessageW(hwnd, firetray.Win32.WM_TRAYMESSAGEFWD, 0, 1);
+        log.debug("SendMessageW WM_TRAYMESSAGEFWD rv="+rv+" winLastError="+ctypes.winLastError);
+      }
+
+  } catch(error) {
+log.error(error);
+  }
+
+    }
 
     return user32.DefWindowProcW(hWnd, uMsg, wParam, lParam);
   },
 
   getIconFromWindow: function(hwnd) {
-    rv = user32.SendMessageW(hwnd, user32.WM_GETICON, user32.ICON_SMALL, 0);
+    let rv = user32.SendMessageW(hwnd, user32.WM_GETICON, user32.ICON_SMALL, 0);
+    log.debug("SendMessageW winLastError="+ctypes.winLastError);
     // result is a ctypes.Int64. So we need to create a CData from it before
     // casting it to a HICON.
     let icon = ctypes.cast(win32.LRESULT(rv), win32.HICON);
     let NULL = win32.HICON(null); // for comparison only
-    log.debug("SendMessageW winLastError="+ctypes.winLastError);
     if (firetray.js.strEquals(icon, NULL)) { // from the window class
       rv = user32.GetClassLong(hwnd, user32.GCLP_HICONSM);
       icon = ctypes.cast(win32.ULONG_PTR(rv), win32.HICON);

src/modules/winnt/FiretrayWin32.jsm

@@ -15,8 +15,8 @@ firetray.Handler.subscribeLibsForClosing([kernel32, user32]);
 
 let log = firetray.Logging.getLogger("firetray.Win32");
 
-const kMessageTray     = "_FIRETRAY_TrayMessage";
-const kMessageCallback = "_FIRETRAY_TrayCallback";
+const kMessageTray     = "_FIRETRAY_Tray";
+const kMessageTrayFwd  = "_FIRETRAY_TrayFwd";
 
 if ("undefined" == typeof(firetray.Handler))
   log.error("This module MUST be imported from/after FiretrayHandler !");
@@ -26,9 +26,10 @@ function Win32Env() {
   this.hInstance = kernel32.GetModuleHandleW("xul"); // ordinary windows are created from xul.dll
   log.debug("hInstance="+this.hInstance);
 
+  // we use our own messages because we create a different window class than Moz
   this.WM_TASKBARCREATED = user32.RegisterWindowMessageW("TaskbarCreated");
-  this.WM_TRAYMESSAGE  = user32.RegisterWindowMessageW(kMessageTray);
-  this.WM_TRAYCALLBACK = user32.RegisterWindowMessageW(kMessageCallback);
+  this.WM_TRAYMESSAGE    = user32.RegisterWindowMessageW(kMessageTray);
+  this.WM_TRAYMESSAGEFWD = user32.RegisterWindowMessageW(kMessageTrayFwd);
   log.debug("WM_*="+this.WM_TASKBARCREATED+" "+this.WM_TRAYMESSAGE+" "+this.WM_TRAYCALLBACK);
 
   /* if Administrator, accept messages from applications running in a lower
@@ -48,6 +49,14 @@ function Win32Env() {
     return rv;
   };
 
+  // wid will be used as a string most of the time (through f.Handler.windows mainly)
+  this.hwndToHexStr = function(hWnd) {
+    return "0x" + ctypes.cast(hWnd, ctypes.uintptr_t).value.toString(16);
+  };
+  this.hexStrToHwnd = function(wid) {
+    return win32.HWND(ctypes.UInt64(wid));
+  };
+
 }
 
 firetray.Win32 = new Win32Env();

src/modules/winnt/FiretrayWindow.jsm

@@ -9,11 +9,12 @@ const Cu = Components.utils;
 Cu.import("resource://gre/modules/ctypes.jsm");
 Cu.import("resource://firetray/ctypes/ctypesMap.jsm");
 Cu.import("resource://firetray/ctypes/winnt/win32.jsm");
+Cu.import("resource://firetray/ctypes/winnt/kernel32.jsm");
 Cu.import("resource://firetray/ctypes/winnt/user32.jsm");
 Cu.import("resource://firetray/winnt/FiretrayWin32.jsm");
 Cu.import("resource://firetray/FiretrayWindow.jsm");
 Cu.import("resource://firetray/commons.js");
-firetray.Handler.subscribeLibsForClosing([user32]);
+firetray.Handler.subscribeLibsForClosing([kernel32, user32]);
 
 let log = firetray.Logging.getLogger("firetray.Window");
 
@@ -28,8 +29,7 @@ const kPropProcPrev = "_FIRETRAY_OLD_PROC";
 // NOTE: storing ctypes pointers into a JS object doesn't work: pointers are
 // "evolving" after a while (maybe due to back and forth conversion). So we
 // need to store them into a real ctypes array !
-firetray.Handler.wndProcs     = new ctypesMap(user32.WNDPROC);
-firetray.Handler.wndProcsOrig = new ctypesMap(user32.WNDPROC);
+firetray.Handler.callProcHooks     = new ctypesMap(win32.HHOOK);
 
 
 firetray.Window = new FiretrayWindow();
@@ -57,11 +57,6 @@ firetray.Window.startupHide = function(xid) {
 firetray.Window.setVisibility = function(xid, visibility) {
 };
 
-// wid will be used as a string most of the time (through f.Handler.windows mainly)
-firetray.Window.hwndToHexStr = function(hWnd) {
-  return "0x" + ctypes.cast(hWnd, ctypes.uintptr_t).value.toString(16);
-};
-
 firetray.Window.wndProc = function(hWnd, uMsg, wParam, lParam) { // filterWindow
   log.debug("wndProc CALLED: hWnd="+hWnd+", uMsg="+uMsg+", wParam="+wParam+", lParam="+lParam);
 
@@ -70,16 +65,12 @@ firetray.Window.wndProc = function(hWnd, uMsg, wParam, lParam) { // filterWindow
 
 try {
 
-  let wid = firetray.Window.hwndToHexStr(hWnd);
+  let wid = firetray.Win32.hwndToHexStr(hWnd);
   // let procPrev = firetray.Handler.wndProcsOrig.get(wid);
   // let procPrev = ctypes.cast(user32.GetPropW(hWnd, win32._T(kPropProcPrev)), user32.WNDPROC);
   // let procPrev = user32.GetPropW(hWnd, win32._T(kPropProcPrev));
   log.debug("  wid="+wid+" prev="+procPrev);
 
-  /*
-   * https://bugzilla.mozilla.org/show_bug.cgi?id=598679
-   * https://bugzilla.mozilla.org/show_bug.cgi?id=671266
-   */
   // let rv = user32.CallWindowProcW(procPrev, hWnd, uMsg, wParam, lParam);
   let rv = procPrev(hWnd, uMsg, wParam, lParam);
   log.debug("  CallWindowProc="+rv);
@@ -100,6 +91,34 @@ log.error(error);
   // return user32.DefWindowProcW(hWnd, uMsg, wParam, lParam);
 };
 
+firetray.Window.callProcHook = function(nCode, wParam, lParam) { // WH_CALLWNDPROC, WH_GETMESSAGE
+  // log.debug("callProcHook CALLED: nCode="+nCode+", wParam="+wParam+", lParam="+lParam);
+  let cwpstruct = ctypes.cast(win32.LPARAM(lParam), user32.CWPSTRUCT.ptr).contents;
+  let uMsg = cwpstruct.message;
+
+  if (uMsg === firetray.Win32.WM_TRAYMESSAGEFWD) {
+    log.debug("WM_TRAYMESSAGEFWD received!");
+
+    if (wParam != 0) {
+      log.debug("message was sent by the current thread");
+    }
+
+    let hWnd = cwpstruct.hwnd;
+    log.debug("hWnd="+hWnd);
+    let wid = firetray.Win32.hwndToHexStr(hWnd);
+    log.debug("wid="+wid);
+
+    if (nCode === user32.HC_ACTION) {
+      log.warn("*** must process the message ***");
+    } else if (nCode < 0) {
+      log.warn("*** must pass the message to the CallNextHookEx function without further processing ***");
+    }
+
+  }
+
+  return user32.CallNextHookEx(null, nCode, wParam, lParam);
+};
+
 
 
 ///////////////////////// firetray.Handler overriding /////////////////////////
@@ -121,7 +140,7 @@ firetray.Handler.registerWindow = function(win) {
   let hwnd = nativeHandle ?
         new ctypes.voidptr_t(ctypes.UInt64(nativeHandle)) :
         user32.FindWindowW("MozillaWindowClass", win.document.title);
-  let wid = firetray.Window.hwndToHexStr(hwnd);
+  let wid = firetray.Win32.hwndToHexStr(hwnd);
   log.debug("=== hwnd="+hwnd+" wid="+wid+" win.document.title: "+win.document.title);
 
   if (this.windows.hasOwnProperty(wid)) {
@@ -144,39 +163,28 @@ firetray.Handler.registerWindow = function(win) {
   // windows *are* shown at startup
   firetray.Window.updateVisibility(wid, true);
   log.debug("window "+wid+" registered");
-  // NOTE: shouldn't be necessary to gtk_widget_add_events(gtkWin, gdk.GDK_ALL_EVENTS_MASK);
 
 /*
-  // try {
-try {
-    let wndProc = user32.WNDPROC(firetray.Window.wndProc);
-    log.debug("proc="+wndProc);
-    this.wndProcs.insert(wid, wndProc);
-    let procPrev = user32.WNDPROC(
-      user32.SetWindowLongW(hwnd, user32.GWLP_WNDPROC, ctypes.cast(wndProc, win32.LONG_PTR))
-    );
-    log.debug("procPrev="+procPrev+" winLastError="+ctypes.winLastError);
-    this.wndProcsOrig.insert(wid, procPrev); // could be set as a window prop (SetPropW)
+  try {
+    let callProcHook = user32.HOOKPROC(firetray.Window.callProcHook);
+    log.debug("callhk="+callProcHook);
+    // Global hooks must reside in a dll (hence hInst). This is important for
+    // the scope of variables.
+    let hhook = user32.SetWindowsHookExW(
+      user32.WH_CALLWNDPROC, callProcHook, null, kernel32.GetCurrentThreadId());
+    log.debug("  hhook="+hhook+" winLastError="+ctypes.winLastError);
+    this.callProcHooks.insert(wid, hhook);
 
-    procPrev = ctypes.cast(procPrev, win32.HANDLE);
-    user32.SetPropW(hwnd, win32._T(kPropProcPrev), procPrev);
-    log.debug("SetPropW: "+procPrev+" winLastError="+ctypes.winLastError);
-  } catch(error) {
-log.error(error);
+    firetray.Win32.acceptAllMessages(hwnd);
+
+  } catch (x) {
+    if (x.name === "RangeError") // instanceof not working :-(
+      win.alert(x+"\n\nYou seem to have more than "+FIRETRAY_WINDOW_COUNT_MAX
+                +" windows open. This breaks FireTray and most probably "
+                +firetray.Handler.appName+".");
+    else win.alert(x);
   }
 */
-    // firetray.Win32.acceptAllMessages(hwnd);
-
-  // } catch (x) {
-  //   if (x.name === "RangeError") // instanceof not working :-(
-  //     win.alert(x+"\n\nYou seem to have more than "+FIRETRAY_WINDOW_COUNT_MAX
-  //               +" windows open. This breaks FireTray and most probably "
-  //               +firetray.Handler.appName+".");
-  //   else win.alert(x);
-  // }
-
-  // TODO: check wndproc chaining http://stackoverflow.com/a/8835843/421846 if
-  // needed for startupFilter
 
   log.debug("AFTER"); firetray.Handler.dumpWindows();
   return wid;
@@ -194,8 +202,10 @@ firetray.Handler.unregisterWindow = function(win) {
 
   if (!delete firetray.Handler.windows[wid])
     throw new DeleteError();
-  // firetray.Handler.wndProcs.remove(wid);
-  // firetray.Handler.wndProcsOrig.remove(wid);
+/*
+  user32.UnhookWindowsHookEx(firetray.Handler.callProcHooks.get(wid));
+  firetray.Handler.callProcHooks.remove(wid);
+*/
   firetray.Handler.windowsCount -= 1;
   firetray.Handler.visibleWindowsCount -= 1;