win32: use native ANSI sequence processing, if possible (#4700)

Windows 10 version 1511 (also known as Anniversary Update), according to
https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
introduced native support for ANSI sequence processing. This allows
using colors from the entire 24-bit color range.

All we need to do is test whether the console's "virtual processing
support" can be enabled. If it can, we do not even need to start the
`console_thread` to handle ANSI sequences.

Incidentally, this addresses
#3712.

Author: dscho

compat/winansi.c

@@ -594,6 +594,49 @@ static void detect_msys_tty(int fd)
 
 #endif
 
+static HANDLE std_console_handle;
+static DWORD std_console_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+static UINT std_console_code_page = CP_UTF8;
+
+static void reset_std_console(void)
+{
+	if (std_console_mode != ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+		SetConsoleMode(std_console_handle, std_console_mode);
+	if (std_console_code_page != CP_UTF8)
+		SetConsoleOutputCP(std_console_code_page);
+}
+
+static int enable_virtual_processing(void)
+{
+	std_console_handle = GetStdHandle(STD_OUTPUT_HANDLE);
+	if (std_console_handle == INVALID_HANDLE_VALUE ||
+	    !GetConsoleMode(std_console_handle, &std_console_mode)) {
+		std_console_handle = GetStdHandle(STD_ERROR_HANDLE);
+		if (std_console_handle == INVALID_HANDLE_VALUE ||
+		    !GetConsoleMode(std_console_handle, &std_console_mode))
+		return 0;
+	}
+
+	std_console_code_page = GetConsoleOutputCP();
+	if (std_console_code_page != CP_UTF8)
+		SetConsoleOutputCP(CP_UTF8);
+	if (!std_console_code_page)
+		std_console_code_page = CP_UTF8;
+
+	atexit(reset_std_console);
+
+	if (std_console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+		return 1;
+
+	if (!SetConsoleMode(std_console_handle,
+			    std_console_mode |
+			    ENABLE_PROCESSED_OUTPUT |
+			    ENABLE_VIRTUAL_TERMINAL_PROCESSING))
+		return 0;
+
+	return 1;
+}
+
 /*
  * Wrapper for isatty().  Most calls in the main git code
  * call isatty(1 or 2) to see if the instance is interactive
@@ -632,6 +675,9 @@ void winansi_init(void)
 		return;
 	}
 
+	if (enable_virtual_processing())
+		return;
+
 	/* create a named pipe to communicate with the console thread */
 	if (swprintf(name, ARRAY_SIZE(name) - 1, L"\\\\.\\pipe\\winansi%lu",
 		     GetCurrentProcessId()) < 0)