Dark mode: themed message boxes and dark scrollbars

Address PR zufuliu#1203 feedback: scroll bars and the save-changes dialog were rendered in the light theme even when dark mode is active.

- Enable scroll bar fix in darkmodelib (mode 2: per-window) and rely on
  the existing enableDarkScrollBarForWindowAndChildren(hwndMain) call so
  the Scintilla edit window's scroll bars use the dark theme.
- Route Notepad4's MsgBox() through dmlib::darkMessageBoxW(), which
  renders message boxes as themed task dialogs when dark mode is on.
- Fix two upstream darkmodelib namespacing bugs that surface when
  _DARKMODELIB_USE_SCROLLBAR_FIX is defined (DmlibHook.cpp uses
  ModuleHandle / LoadFn without the dmlib_module:: prefix).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: lostindark

build/VisualStudio/Notepad4.vcxproj

@@ -158,7 +158,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <AdditionalIncludeDirectories>..\..\scintilla\include;..\..\scintilla\lexlib;..\..\scintilla\src;..\..\src;..\..\darkmodelib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_WINDOWS;NOMINMAX;WIN32_LEAN_AND_MEAN;STRICT_TYPED_ITEMIDS;BOOST_REGEX_STANDALONE;NO_CXX11_REGEX;UNICODE;_UNICODE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DARKMODELIB_NO_INI_CONFIG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_WINDOWS;NOMINMAX;WIN32_LEAN_AND_MEAN;STRICT_TYPED_ITEMIDS;BOOST_REGEX_STANDALONE;NO_CXX11_REGEX;UNICODE;_UNICODE;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DARKMODELIB_NO_INI_CONFIG;_DARKMODELIB_USE_SCROLLBAR_FIX=2;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <WarningLevel>Level4</WarningLevel>
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <LanguageStandard_C>stdc17</LanguageStandard_C>

build/mingw/notepad4.mk

@@ -14,7 +14,7 @@ INCDIR = \
 	-I"$(scintilla_dir)/include" \
 	-I"$(darkmodelib_dir)"
 
-CPPFLAGS += -D_DARKMODELIB_NO_INI_CONFIG
+CPPFLAGS += -D_DARKMODELIB_NO_INI_CONFIG -D_DARKMODELIB_USE_SCROLLBAR_FIX=2
 
 LDFLAGS += -L"$(BINFOLDER)/obj"
 

darkmodelib/DmlibHook.cpp

@@ -23,6 +23,7 @@
 
 #if defined(_DARKMODELIB_USE_SCROLLBAR_FIX) && (_DARKMODELIB_USE_SCROLLBAR_FIX > 0)
 #include <mutex>
+#include <string_view>
 #include <unordered_set>
 #endif
 
@@ -168,7 +169,7 @@ static fnOpenNcThemeData pfOpenNcThemeData = nullptr;
 
 bool dmlib_hook::loadOpenNcThemeData(const HMODULE& hUxtheme) noexcept
 {
-	return LoadFn(hUxtheme, pfOpenNcThemeData, 49);
+	return dmlib_module::LoadFn(hUxtheme, pfOpenNcThemeData, 49);
 }
 
 #if defined(_DARKMODELIB_USE_SCROLLBAR_FIX) && (_DARKMODELIB_USE_SCROLLBAR_FIX > 1)
@@ -214,7 +215,7 @@ static bool isWindowOrParentUsingDarkScrollBar(HWND hWnd)
 static HTHEME WINAPI MyOpenNcThemeData(HWND hWnd, LPCWSTR pszClassList)
 {
 	static constexpr std::wstring_view scrollBarClassName = WC_SCROLLBAR;
-	if (scrollBarClassName == pszClassList)
+	if (scrollBarClassName == pszClassList && dmlib_win32api::IsDarkModeActive())
 	{
 #if defined(_DARKMODELIB_USE_SCROLLBAR_FIX) && (_DARKMODELIB_USE_SCROLLBAR_FIX > 1)
 		if (isWindowOrParentUsingDarkScrollBar(hWnd))
@@ -229,7 +230,7 @@ static HTHEME WINAPI MyOpenNcThemeData(HWND hWnd, LPCWSTR pszClassList)
 
 void dmlib_hook::fixDarkScrollBar()
 {
-	const ModuleHandle moduleComctl(L"comctl32.dll");
+	const dmlib_module::ModuleHandle moduleComctl(L"comctl32.dll");
 	if (moduleComctl.isLoaded())
 	{
 		auto* addr = iat_hook::FindDelayLoadThunkInModule(moduleComctl.get(), "uxtheme.dll", 49); // OpenNcThemeData

src/DarkMode.cpp

@@ -10,14 +10,21 @@
 
 // Declared in Styles.cpp
 extern int np2StyleTheme;
+// Declared in Notepad4.cpp - this app's module handle
+extern HINSTANCE g_hInstance;
 
 // Thread-level hook to intercept WM_INITDIALOG and apply dark mode to dialogs
 static HHOOK s_hCallWndProcRetHook = nullptr;
 
 static LRESULT CALLBACK DarkMode_CallWndRetProc(int nCode, WPARAM wParam, LPARAM lParam) noexcept {
 	if (nCode == HC_ACTION) {
 		const CWPRETSTRUCT *cwpret = reinterpret_cast<const CWPRETSTRUCT *>(lParam);
-		if (cwpret->message == WM_INITDIALOG) {
+		// Only theme dialogs created by this app while dark mode is active.
+		// This avoids touching system dialogs (e.g. the common file open/save
+		// dialog) that are hosted in our process but owned by comdlg32/shell.
+		if (cwpret->message == WM_INITDIALOG
+			&& dmlib::isExperimentalActive()
+			&& reinterpret_cast<HINSTANCE>(GetWindowLongPtr(cwpret->hwnd, GWLP_HINSTANCE)) == g_hInstance) {
 			DarkMode_ApplyToDialog(cwpret->hwnd);
 		}
 	}
@@ -113,10 +120,36 @@ void DarkMode_OnThemeChanged(int newTheme) noexcept {
 	}
 }
 
+// Broadcast WM_THEMECHANGED to a window and all its descendants so that
+// themed controls (including scroll bars) re-open their theme handles.
+static BOOL CALLBACK DarkMode_SendThemeChangedProc(HWND hwnd, LPARAM /*lParam*/) noexcept {
+	SendMessage(hwnd, WM_THEMECHANGED, 0, 0);
+	return TRUE;
+}
+
+void DarkMode_BroadcastThemeChanged(HWND hwnd) noexcept {
+	if (hwnd == nullptr) {
+		return;
+	}
+	SendMessage(hwnd, WM_THEMECHANGED, 0, 0);
+	EnumChildWindows(hwnd, DarkMode_SendThemeChangedProc, 0);
+}
+
 bool DarkMode_HandleSettingChange([[maybe_unused]] HWND hwnd, LPARAM lParam) noexcept {
 	return dmlib::handleSettingChange(lParam);
 }
 
 bool DarkMode_IsEnabled() noexcept {
 	return dmlib::isExperimentalActive();
 }
+
+int DarkMode_MessageBox(HWND hwnd, LPCWSTR text, LPCWSTR caption, UINT uType, WORD wLanguageId) noexcept {
+	if (dmlib::isExperimentalActive()) {
+		const HRESULT hr = dmlib::darkMessageBoxW(hwnd, text, caption, uType);
+		if (hr > 0) {
+			return static_cast<int>(hr);
+		}
+		// Fall through to MessageBoxEx on failure.
+	}
+	return MessageBoxEx(hwnd, text, caption, uType, wLanguageId);
+}

src/DarkMode.h

@@ -28,3 +28,11 @@ bool DarkMode_HandleSettingChange(HWND hwnd, LPARAM lParam) noexcept;
 
 // Check if dark mode UI is currently active.
 bool DarkMode_IsEnabled() noexcept;
+
+// Show a message box that respects dark mode (uses Task Dialog when dark mode is active).
+// Returns the same kind of value as MessageBoxEx (IDOK, IDYES, IDNO, IDCANCEL, ...).
+int DarkMode_MessageBox(HWND hwnd, LPCWSTR text, LPCWSTR caption, UINT uType, WORD wLanguageId) noexcept;
+
+// Broadcast WM_THEMECHANGED to the window and all its descendants so that
+// themed controls (including scroll bars) re-open their theme handles.
+void DarkMode_BroadcastThemeChanged(HWND hwnd) noexcept;

src/Dialogs.cpp

@@ -34,6 +34,7 @@
 #include "Styles.h"
 #include "Dlapi.h"
 #include "Dialogs.h"
+#include "DarkMode.h"
 #include "resource.h"
 #include "Version.h"
 
@@ -114,7 +115,7 @@ int MsgBox(UINT uType, UINT uIdMsg, ...) noexcept {
 
 	HWND hwnd = GetMsgBoxParent();
 	PostMessage(hwndMain, APPM_CENTER_MESSAGE_BOX, AsInteger<WPARAM>(hwnd), 0);
-	return MessageBoxEx(hwnd, szText, szTitle, uType, lang);
+	return DarkMode_MessageBox(hwnd, szText, szTitle, uType, lang);
 }
 
 //=============================================================================

src/Notepad4.cpp

@@ -1992,6 +1992,10 @@ void CreateBars(HWND hwnd, HINSTANCE hInstance) noexcept {
 	dmlib::setDarkExplorerTheme(hwndToolbar);
 	dmlib::setStatusBarCtrlSubclass(hwndStatus);
 	dmlib::setWindowEraseBgSubclass(hwndReBar);
+	// Subclass the main window so darkmodelib can custom-draw toolbar buttons
+	// (dark hover/checked states) via the toolbar's NM_CUSTOMDRAW notifications,
+	// which the rebar forwards up to its parent.
+	dmlib::setWindowNotifyCustomDrawSubclass(hwnd);
 	dmlib::enableDarkScrollBarForWindowAndChildren(hwnd);
 
 	// Theme toolbar tooltips

src/Styles.cpp

@@ -1201,6 +1201,9 @@ void Style_OnStyleThemeChanged(int theme) noexcept {
 	DarkMode_OnThemeChanged(theme);
 	if (hwndMain) {
 		DarkMode_ApplyToWindow(hwndMain);
+		// Make controls (including scroll bars) re-open theme handles so
+		// they pick up the new light/dark scroll bar style.
+		DarkMode_BroadcastThemeChanged(hwndMain);
 		RedrawWindow(hwndMain, nullptr, nullptr, RDW_INVALIDATE | RDW_ERASE | RDW_ALLCHILDREN | RDW_UPDATENOW | RDW_FRAME);
 	}