JRE-1228 support custom frame decorationjb8u202-b1514.4

(cherry picked from commit 3319fcbdfa7f1ad5c0a013f549b53925e9d1e51e)

16 files changed, 162 insertions, 13 deletions

diff --git a/src/share/classes/java/awt/Window.java b/src/share/classes/java/awt/Window.java
index 947c945e74..6654a75b0c 100644
--- a/src/share/classes/java/awt/Window.java
+++ b/src/share/classes/java/awt/Window.java
@@ -3936,6 +3936,18 @@ public class Window extends Container implements Accessible {
         }
     }
 
+    private volatile boolean hasCustomDecoration;
+
+    boolean hasCustomDecoration() {
+        return hasCustomDecoration;
+    }
+
+    /**
+     * Set via reflection (JB JdkEx API).
+     */
+    void setHasCustomDecoration() {
+        hasCustomDecoration = true;
+    }
 
     // ************************** MIXING CODE *******************************
 
diff --git a/src/windows/classes/sun/awt/windows/WWindowPeer.java b/src/windows/classes/sun/awt/windows/WWindowPeer.java
index 8359db8f57..64768902f5 100644
--- a/src/windows/classes/sun/awt/windows/WWindowPeer.java
+++ b/src/windows/classes/sun/awt/windows/WWindowPeer.java
@@ -36,6 +36,7 @@ import java.beans.*;
 
 import java.util.*;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 import sun.java2d.SunGraphics2D;
 import sun.util.logging.PlatformLogger;
@@ -996,4 +997,18 @@ public class WWindowPeer extends WPanelPeer implements WindowPeer,
         }
         return err;
     }
+
+    private volatile List<Rectangle> hitTestSpots;
+
+    private void setCustomDecorationHitTestSpots(List<Rectangle> hitTestSpots) {
+        this.hitTestSpots = new CopyOnWriteArrayList<>(hitTestSpots);
+    }
+
+    private boolean hitTestCustomDecoration(int x, int y) {
+        if (hitTestSpots == null) return false;
+        for (Rectangle spot : hitTestSpots) {
+            if (spot.contains(x, y)) return true;
+        }
+        return false;
+    }
 }
diff --git a/src/windows/native/sun/windows/awt_Component.cpp b/src/windows/native/sun/windows/awt_Component.cpp
index 227694bd70..c7bb7aaefa 100644
--- a/src/windows/native/sun/windows/awt_Component.cpp
+++ b/src/windows/native/sun/windows/awt_Component.cpp
@@ -1951,7 +1951,7 @@ LRESULT AwtComponent::WindowProc(UINT message, WPARAM wParam, LPARAM lParam)
           mr = WmNcPaint((HRGN)wParam);
           break;
       case WM_NCHITTEST:
-          mr = WmNcHitTest(LOWORD(lParam), HIWORD(lParam), retValue);
+          mr = WmNcHitTest(GET_X_LPARAM(lParam), GET_Y_LPARAM(lParam), retValue);
           break;
 
       case WM_AWT_RESHAPE_COMPONENT: {
@@ -4676,7 +4676,7 @@ MsgRouting AwtComponent::WmNcPaint(HRGN hrgn)
     return mrDoDefault;
 }
 
-MsgRouting AwtComponent::WmNcHitTest(UINT x, UINT y, LRESULT &retVal)
+MsgRouting AwtComponent::WmNcHitTest(int x, int y, LRESULT &retVal)
 {
     return mrDoDefault;
 }
diff --git a/src/windows/native/sun/windows/awt_Component.h b/src/windows/native/sun/windows/awt_Component.h
index 56d1b4d357..b0f5043760 100644
--- a/src/windows/native/sun/windows/awt_Component.h
+++ b/src/windows/native/sun/windows/awt_Component.h
@@ -583,7 +583,7 @@ public:
                                     LPNCCALCSIZE_PARAMS lpncsp,
                                     LRESULT &retVal);
     virtual MsgRouting WmNcPaint(HRGN hrgn);
-    virtual MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT &retVal);
+    virtual MsgRouting WmNcHitTest(int x, int y, LRESULT &retVal);
     virtual MsgRouting WmSysCommand(UINT uCmdType, int xPos, int yPos);
     virtual MsgRouting WmExitSizeMove();
     virtual MsgRouting WmEnterMenuLoop(BOOL isTrackPopupMenu);
diff --git a/src/windows/native/sun/windows/awt_Frame.cpp b/src/windows/native/sun/windows/awt_Frame.cpp
index 371c22fcc1..ac8ceb4302 100644
--- a/src/windows/native/sun/windows/awt_Frame.cpp
+++ b/src/windows/native/sun/windows/awt_Frame.cpp
@@ -32,6 +32,8 @@
 #include "ComCtl32Util.h"
 
 #include <windowsx.h>
+#include <uxtheme.h>
+#include <dwmapi.h>
 
 #include <java_lang_Integer.h>
 #include <sun_awt_windows_WEmbeddedFrame.h>
@@ -126,6 +128,7 @@ AwtFrame::AwtFrame() {
     m_zoomed = FALSE;
     m_maxBoundsSet = FALSE;
     m_forceResetZoomed = FALSE;
+    m_pHasCustomDecoration = NULL;
 
     isInManualMoveOrSize = FALSE;
     grabbedHitTest = 0;
@@ -1696,6 +1699,118 @@ ret:
     delete nmbs;
 }
 
+// {start} Custom Decoration Support
+
+BOOL AwtFrame::HasCustomDecoration()
+{
+    if (!m_pHasCustomDecoration) {
+        m_pHasCustomDecoration = new BOOL;
+        JNIEnv *env = (JNIEnv *) JNU_GetEnv(jvm, JNI_VERSION_1_2);
+        *m_pHasCustomDecoration = JNU_CallMethodByName(env, NULL, GetTarget(env), "hasCustomDecoration", "()Z").z;
+    }
+    return *m_pHasCustomDecoration;
+}
+
+void GetSysInsets(RECT* insets) {
+    static RECT* sysInsets = NULL;
+
+    if (!sysInsets) {
+        sysInsets = new RECT;
+        sysInsets->left = sysInsets->right = ::GetSystemMetrics(SM_CXSIZEFRAME);
+        sysInsets->top = sysInsets->bottom = ::GetSystemMetrics(SM_CYSIZEFRAME);
+        sysInsets->top += ::GetSystemMetrics(SM_CYCAPTION);
+    }
+    ::CopyRect(insets, sysInsets);
+}
+
+LRESULT HitTestNCA(AwtFrame* frame, int x, int y) {
+    RECT rcWindow;
+    RECT insets;
+
+    GetSysInsets(&insets);
+    GetWindowRect(frame->GetHWnd(), &rcWindow);
+
+    // Get the frame rectangle, adjusted for the style without a caption.
+    RECT rcFrame = {0};
+    AdjustWindowRectEx(&rcFrame, WS_OVERLAPPEDWINDOW & ~WS_CAPTION, FALSE, NULL);
+
+    USHORT uRow = 1;
+    USHORT uCol = 1;
+    BOOL fOnResizeBorder = FALSE;
+
+    if (y >= rcWindow.top &&
+        y < rcWindow.top + insets.top)
+    {
+        JNIEnv *env = (JNIEnv *) JNU_GetEnv(jvm, JNI_VERSION_1_2);
+        if (JNU_CallMethodByName(env, NULL, frame->GetPeer(env),
+                                 "hitTestCustomDecoration", "(II)Z",
+                                 frame->ScaleDownX(x - rcWindow.left),
+                                 frame->ScaleDownY(y - rcWindow.top)).z)
+        {
+            return HTNOWHERE;
+        }
+        fOnResizeBorder = (y < (rcWindow.top - rcFrame.top));
+        uRow = 0;
+    } else if (y < rcWindow.bottom &&
+               y >= rcWindow.bottom - insets.bottom) {
+        uRow = 2;
+    }
+
+    if (x >= rcWindow.left &&
+        x < rcWindow.left + insets.left)
+    {
+        uCol = 0;
+    } else if (x < rcWindow.right &&
+               x >= rcWindow.right - insets.right)
+    {
+        uCol = 2;
+    }
+
+    LRESULT hitTests[3][3] = {
+            {HTTOPLEFT, fOnResizeBorder ? HTTOP : HTCAPTION, HTTOPRIGHT},
+            {HTLEFT, HTNOWHERE, HTRIGHT},
+            {HTBOTTOMLEFT, HTBOTTOM, HTBOTTOMRIGHT},
+    };
+
+    return hitTests[uRow][uCol];
+}
+
+MsgRouting AwtFrame::WmNcCalcSize(BOOL wParam, LPNCCALCSIZE_PARAMS lpncsp, LRESULT& retVal)
+{
+    if (!wParam || !HasCustomDecoration()) {
+        return AwtWindow::WmNcCalcSize(wParam, lpncsp, retVal);
+    }
+    if (::IsZoomed(GetHWnd())) {
+        RECT insets;
+        GetSysInsets(&insets);
+        static int xBorder = ::GetSystemMetrics(SM_CXBORDER);
+        static int yBorder = ::GetSystemMetrics(SM_CYBORDER);
+
+        // When maximized we should include insets or otherwise the client area edges get out of a screen
+        lpncsp->rgrc[0].left = lpncsp->rgrc[0].left + insets.left - xBorder;
+        lpncsp->rgrc[0].top = lpncsp->rgrc[0].top + insets.bottom - yBorder; // do not count caption
+        lpncsp->rgrc[0].right = lpncsp->rgrc[0].right - insets.right + xBorder;
+        lpncsp->rgrc[0].bottom = lpncsp->rgrc[0].bottom - insets.bottom + yBorder;
+    }
+    retVal = 0L;
+    return mrConsume;
+}
+
+MsgRouting AwtFrame::WmNcHitTest(int x, int y, LRESULT& retVal)
+{
+    if (!HasCustomDecoration()) {
+        return AwtWindow::WmNcHitTest(x, y, retVal);
+    }
+    if (::IsWindow(GetModalBlocker(GetHWnd()))) {
+        retVal = HTCLIENT;
+        return mrConsume;
+    }
+    retVal = HitTestNCA(this, x, y);
+    return retVal == HTNOWHERE ? mrDoDefault : mrConsume;
+}
+
+// {end} Custom Decoration Support
+
 /************************************************************************
  * WFramePeer native methods
  */
diff --git a/src/windows/native/sun/windows/awt_Frame.h b/src/windows/native/sun/windows/awt_Frame.h
index b10591438a..02a2e74576 100644
--- a/src/windows/native/sun/windows/awt_Frame.h
+++ b/src/windows/native/sun/windows/awt_Frame.h
@@ -117,6 +117,8 @@ public:
     MsgRouting WmGetIcon(WPARAM iconType, LRESULT& retVal);
     MsgRouting WmShowWindow(BOOL show, UINT status);
     MsgRouting WmDPIChanged(UINT xDPI, UINT yDPI, RECT* bounds);
+    MsgRouting WmNcCalcSize(BOOL fCalcValidRects, LPNCCALCSIZE_PARAMS lpncsp, LRESULT& retVal);
+    MsgRouting WmNcHitTest(int x, int y, LRESULT& retVal);
 
     virtual MsgRouting WmSysCommand(UINT uCmdType, int xPos, int yPos);
 
@@ -161,6 +163,9 @@ public:
     INLINE HWND GetImeTargetComponent() { return m_imeTargetComponent; }
     INLINE void SetImeTargetComponent(HWND hwnd) { m_imeTargetComponent = hwnd; }
 
+    BOOL* m_pHasCustomDecoration;
+    BOOL HasCustomDecoration();
+
 protected:
     /* The frame is undecorated. */
     BOOL m_isUndecorated;
diff --git a/src/windows/native/sun/windows/awt_List.cpp b/src/windows/native/sun/windows/awt_List.cpp
index d5b1a38866..d504231a75 100644
--- a/src/windows/native/sun/windows/awt_List.cpp
+++ b/src/windows/native/sun/windows/awt_List.cpp
@@ -412,7 +412,7 @@ AwtList::OwnerMeasureItem(UINT /*ctrlId*/, MEASUREITEMSTRUCT& measureInfo)
 }
 
 MsgRouting
-AwtList::WmNcHitTest(UINT x, UINT y, LRESULT& retVal)
+AwtList::WmNcHitTest(int x, int y, LRESULT& retVal)
 {
     if (::IsWindow(AwtWindow::GetModalBlocker(AwtComponent::GetTopLevelParentForWindow(GetHWnd())))) {
         retVal = HTCLIENT;
diff --git a/src/windows/native/sun/windows/awt_List.h b/src/windows/native/sun/windows/awt_List.h
index 73effc3a56..17a617a5c4 100644
--- a/src/windows/native/sun/windows/awt_List.h
+++ b/src/windows/native/sun/windows/awt_List.h
@@ -109,7 +109,7 @@ public:
     /*
      * Windows message handler functions
      */
-    MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT& retVal);
+    MsgRouting WmNcHitTest(int x, int y, LRESULT& retVal);
     MsgRouting WmMouseDown(UINT flags, int x, int y, int button);
     MsgRouting WmMouseUp(UINT flags, int x, int y, int button);
     MsgRouting WmNotify(UINT notifyCode);
diff --git a/src/windows/native/sun/windows/awt_ScrollPane.cpp b/src/windows/native/sun/windows/awt_ScrollPane.cpp
index 1e4091bd50..9c1ef439e9 100644
--- a/src/windows/native/sun/windows/awt_ScrollPane.cpp
+++ b/src/windows/native/sun/windows/awt_ScrollPane.cpp
@@ -359,7 +359,7 @@ void AwtScrollPane::PostScrollEvent(int orient, int scrollCode, int pos) {
 }
 
 MsgRouting
-AwtScrollPane::WmNcHitTest(UINT x, UINT y, LRESULT& retVal)
+AwtScrollPane::WmNcHitTest(int x, int y, LRESULT& retVal)
 {
     if (::IsWindow(AwtWindow::GetModalBlocker(AwtComponent::GetTopLevelParentForWindow(GetHWnd())))) {
         retVal = HTCLIENT;
diff --git a/src/windows/native/sun/windows/awt_ScrollPane.h b/src/windows/native/sun/windows/awt_ScrollPane.h
index 129b0e12cd..203ef348d5 100644
--- a/src/windows/native/sun/windows/awt_ScrollPane.h
+++ b/src/windows/native/sun/windows/awt_ScrollPane.h
@@ -75,7 +75,7 @@ public:
     /*
      * Windows message handler functions
      */
-    virtual MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT& retVal);
+    virtual MsgRouting WmNcHitTest(int x, int y, LRESULT& retVal);
     virtual MsgRouting WmHScroll(UINT scrollCode, UINT pos, HWND hScrollBar);
     virtual MsgRouting WmVScroll(UINT scrollCode, UINT pos, HWND hScrollBar);
 
diff --git a/src/windows/native/sun/windows/awt_Scrollbar.cpp b/src/windows/native/sun/windows/awt_Scrollbar.cpp
index 502b9a0d23..aa5c6a9a51 100644
--- a/src/windows/native/sun/windows/awt_Scrollbar.cpp
+++ b/src/windows/native/sun/windows/awt_Scrollbar.cpp
@@ -223,7 +223,7 @@ AwtScrollbar::WindowProc(UINT message, WPARAM wParam, LPARAM lParam)
 }
 
 MsgRouting
-AwtScrollbar::WmNcHitTest(UINT x, UINT y, LRESULT& retVal)
+AwtScrollbar::WmNcHitTest(int x, int y, LRESULT& retVal)
 {
     if (::IsWindow(AwtWindow::GetModalBlocker(AwtComponent::GetTopLevelParentForWindow(GetHWnd())))) {
         retVal = HTCLIENT;
diff --git a/src/windows/native/sun/windows/awt_Scrollbar.h b/src/windows/native/sun/windows/awt_Scrollbar.h
index 9369cab0e5..6db58abf8f 100644
--- a/src/windows/native/sun/windows/awt_Scrollbar.h
+++ b/src/windows/native/sun/windows/awt_Scrollbar.h
@@ -71,7 +71,7 @@ public:
 
     // Prevent KB Q102552 race.
     virtual MsgRouting WmMouseDown(UINT flags, int x, int y, int button);
-    virtual MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT& retVal);
+    virtual MsgRouting WmNcHitTest(int x, int y, LRESULT& retVal);
 
     virtual MsgRouting HandleEvent(MSG *msg, BOOL synthetic);
 
diff --git a/src/windows/native/sun/windows/awt_TextArea.cpp b/src/windows/native/sun/windows/awt_TextArea.cpp
index 86ad40ba73..d9ee2ab311 100644
--- a/src/windows/native/sun/windows/awt_TextArea.cpp
+++ b/src/windows/native/sun/windows/awt_TextArea.cpp
@@ -138,7 +138,7 @@ AwtTextArea::WindowProc(UINT message, WPARAM wParam, LPARAM lParam) {
 }
 
 MsgRouting
-AwtTextArea::WmNcHitTest(UINT x, UINT y, LRESULT& retVal)
+AwtTextArea::WmNcHitTest(int x, int y, LRESULT& retVal)
 {
     if (::IsWindow(AwtWindow::GetModalBlocker(AwtComponent::GetTopLevelParentForWindow(GetHWnd())))) {
         retVal = HTCLIENT;
diff --git a/src/windows/native/sun/windows/awt_TextArea.h b/src/windows/native/sun/windows/awt_TextArea.h
index 1738e64b8e..eba97e3330 100644
--- a/src/windows/native/sun/windows/awt_TextArea.h
+++ b/src/windows/native/sun/windows/awt_TextArea.h
@@ -59,7 +59,7 @@ public:
     LRESULT WindowProc(UINT message, WPARAM wParam, LPARAM lParam);
 
     MsgRouting WmEnable(BOOL fEnabled);
-    MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT &retVal);
+    MsgRouting WmNcHitTest(int x, int y, LRESULT &retVal);
     MsgRouting HandleEvent(MSG *msg, BOOL synthetic);
 
     virtual BOOL InheritsNativeMouseWheelBehavior();
diff --git a/src/windows/native/sun/windows/awt_Window.cpp b/src/windows/native/sun/windows/awt_Window.cpp
index c085db5354..6e948f8502 100644
--- a/src/windows/native/sun/windows/awt_Window.cpp
+++ b/src/windows/native/sun/windows/awt_Window.cpp
@@ -2008,7 +2008,7 @@ MsgRouting AwtWindow::WmNcCalcSize(BOOL fCalcValidRects,
     return mrRetVal;
 }
 
-MsgRouting AwtWindow::WmNcHitTest(UINT x, UINT y, LRESULT& retVal)
+MsgRouting AwtWindow::WmNcHitTest(int x, int y, LRESULT& retVal)
 {
     // If this window is blocked by modal dialog, return HTCLIENT for any point of it.
     // That prevents it to be moved or resized using the mouse. Disabling these
diff --git a/src/windows/native/sun/windows/awt_Window.h b/src/windows/native/sun/windows/awt_Window.h
index cc24d73e4f..eeb54b4e74 100644
--- a/src/windows/native/sun/windows/awt_Window.h
+++ b/src/windows/native/sun/windows/awt_Window.h
@@ -180,7 +180,7 @@ public:
     virtual MsgRouting WmSettingChange(UINT wFlag, LPCTSTR pszSection);
     virtual MsgRouting WmNcCalcSize(BOOL fCalcValidRects,
                                     LPNCCALCSIZE_PARAMS lpncsp, LRESULT& retVal);
-    virtual MsgRouting WmNcHitTest(UINT x, UINT y, LRESULT& retVal);
+    virtual MsgRouting WmNcHitTest(int x, int y, LRESULT& retVal);
     virtual MsgRouting WmNcMouseDown(WPARAM hitTest, int x, int y, int button);
     virtual MsgRouting WmGetIcon(WPARAM iconType, LRESULT& retVal);
     virtual LRESULT WindowProc(UINT message, WPARAM wParam, LPARAM lParam);
@@ -267,6 +267,8 @@ public:
     inline HWND GetOverriddenHWnd() { return m_overriddenHwnd; }
     inline void OverrideHWnd(HWND hwnd) { m_overriddenHwnd = hwnd; }
 
+    virtual BOOL HasCustomDecoration() { return FALSE; }
+
 private:
     static int ms_instanceCounter;
     static HHOOK ms_hCBTFilter;