From 2103fc66e2adb4bbe9fac9e60803305b45799f12 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 06:31:47 -0700 Subject: [PATCH 01/23] msvc: fix dependencies of compat/msvc.c The file compat/msvc.c includes compat/mingw.c, which means that we have to recompile compat/msvc.o if compat/mingw.c changes. Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 55ff261f87c18c..4e43e83551f0fe 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -396,6 +396,8 @@ else BASIC_CFLAGS += -Zi -MDd endif X = .exe + +compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From eb37e167188675dbf50337ed9dfe048dbc67df2e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 08:38:17 -0400 Subject: [PATCH 02/23] msvc: ignore VS2015 trash files Signed-off-by: Jeff Hostetler --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9f88b94d9c53cf..b3770bda8d5e84 100644 --- a/.gitignore +++ b/.gitignore @@ -218,6 +218,8 @@ *.user *.idb *.pdb +*.ilk +.vs/ /Debug/ /Release/ /.vagrant/ From 0e3819ae03a5229a90397672eee03a86c47b8d59 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 11:32:01 -0400 Subject: [PATCH 03/23] msvc: add NuGet scripts for building with VS2015 This commit contains a GNU Makefile and NuGet configuration scripts to download and install the various third-party libraries that we will need to build/link with when using VS2015 to build Git. The file "compat/vcbuild/README_VS2015.txt" contains instructions for using this. In this commit, "compat/vcbuild/Makefile" contains hard-coded version numbers of the packages we require. These are set to the current versions as of the time of this commit. We use "nuget restore" to install them explicitly using a "package.config". A future improvement would try to use some of the automatic package management functions and eliminate the need to specify exact versions. I tried, but could not get this to work. NuGet was happy dowload "minimum requirements" rather than "lastest" for dependencies -- and only look at one package at a time. For example, both curl and openssl depend upon zlib and have different minimums. It was unclear which version of zlib would be installed and seemed to be dependent on the order of the top-level pacakges. So, I'm skipping that for now. Signed-off-by: Jeff Hostetler --- compat/vcbuild/.gitignore | 1 + compat/vcbuild/Makefile | 122 +++++++++++++++++++++++++++++++ compat/vcbuild/README_VS2015.txt | 58 +++++++++++++++ compat/vcbuild/nuget.config | 27 +++++++ compat/vcbuild/packages.config | 23 ++++++ 5 files changed, 231 insertions(+) create mode 100644 compat/vcbuild/.gitignore create mode 100644 compat/vcbuild/Makefile create mode 100644 compat/vcbuild/README_VS2015.txt create mode 100644 compat/vcbuild/nuget.config create mode 100644 compat/vcbuild/packages.config diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore new file mode 100644 index 00000000000000..e14691508ec6b5 --- /dev/null +++ b/compat/vcbuild/.gitignore @@ -0,0 +1 @@ +GEN.* diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile new file mode 100644 index 00000000000000..b94bcd119a569b --- /dev/null +++ b/compat/vcbuild/Makefile @@ -0,0 +1,122 @@ +## Makefile to install nuget package dependencies. +################################################################## + +INST=GEN.DEPS +INST_INC=$(INST)/include +INST_LIB=$(INST)/lib +INST_BIN=$(INST)/bin + +PKGDIR=GEN.PKGS + +################################################################## +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib + +unpack: + [ -d $(PKGDIR) ] || mkdir $(PKGDIR) + nuget.exe restore packages.config -ConfigFile nuget.config \ + -NonInteractive -OutputDirectory $(PKGDIR) + +insdir: + [ -d $(INST) ] || mkdir $(INST) + [ -d $(INST_INC) ] || mkdir $(INST_INC) + [ -d $(INST_BIN) ] || mkdir $(INST_BIN) + [ -d $(INST_LIB) ] || mkdir $(INST_LIB) + +################################################################## +## We place the expat headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/expat + +EXPAT_VER=2.1.0.11 +EXPAT_ROOT=$(PKGDIR)/expat.$(EXPAT_VER)/build/native +EXPAT_INC=$(EXPAT_ROOT)/include +EXPAT_LIB=$(EXPAT_ROOT)/lib/v110/x64/Release/dynamic/utf8 + +expat: insdir + [ -d $(INST_INC)/expat ] || mkdir $(INST_INC)/expat + cp -r $(EXPAT_INC)/* $(INST_INC)/expat/ + cp -r $(EXPAT_LIB)/* $(INST_LIB)/ + + +################################################################## + +LIBSSH_VER=1.4.3.3 +LIBSSH_ROOT=$(PKGDIR)/libssh2.$(LIBSSH_VER)/build/native +LIBSSH_INC=$(LIBSSH_ROOT)/include +LIBSSH_LIB=$(LIBSSH_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libssh: insdir + [ -d $(INST_INC)/libssh2 ] || mkdir $(INST_INC)/libssh2 + cp -r $(LIBSSH_INC)/* $(INST_INC)/libssh2 + cp -r $(LIBSSH_LIB)/* $(INST_LIB)/ + + +LIBSSH_REDIST_ROOT=$(PKGDIR)/libssh2.redist.$(LIBSSH_VER)/build/native +LIBSSH_REDIST_BIN=$(LIBSSH_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libssh_redist: insdir + cp -r $(LIBSSH_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the curl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +CURL_VER=7.30.0.2 +CURL_ROOT=$(PKGDIR)/curl.$(CURL_VER)/build/native +CURL_INC=$(CURL_ROOT)/include/curl +CURL_LIB=$(CURL_ROOT)/lib/v110/x64/Release/dynamic + +curl: insdir + [ -d $(INST_INC)/curl ] || mkdir $(INST_INC)/curl + cp -r $(CURL_INC)/* $(INST_INC)/curl + cp -r $(CURL_LIB)/* $(INST_LIB)/ + + +CURL_REDIST_ROOT=$(PKGDIR)/curl.redist.$(CURL_VER)/build/native +CURL_REDIST_BIN=$(CURL_REDIST_ROOT)/bin/v110/x64/Release/dynamic + +curl_redist: insdir + cp -r $(CURL_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the openssl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +OPENSSL_VER=1.0.2.1 +OPENSSL_ROOT=$(PKGDIR)/openssl.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$(OPENSSL_VER) +OPENSSL_INC=$(OPENSSL_ROOT)/build/native/include/openssl +OPENSSL_LIB=$(OPENSSL_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/release + +openssl: insdir + [ -d $(INST_INC)/openssl ] || mkdir $(INST_INC)/openssl + cp -r $(OPENSSL_INC)/* $(INST_INC)/openssl + cp -r $(OPENSSL_LIB)/*.lib $(INST_LIB)/ + cp -r $(OPENSSL_LIB)/*.dll $(INST_BIN)/ + cp -r $(OPENSSL_LIB)/*.pdb $(INST_BIN)/ + +################################################################## +## We place the zlib headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/zlib + +ZLIB_VER=1.2.8.8 +ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) +ZLIB_INC=$(ZLIB_ROOT)/build/native/include +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release + +zlib: insdir + [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib + cp -r $(ZLIB_INC)/* $(INST_INC)/zlib + cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ + cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + +################################################################## +clean: + rm -rf $(INST) + +clobber: clean + rm -rf $(PKGDIR) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt new file mode 100644 index 00000000000000..763523fbb4ecda --- /dev/null +++ b/compat/vcbuild/README_VS2015.txt @@ -0,0 +1,58 @@ +Instructions for building Git for Windows using VS2015. +================================================================ + +Installing third-party dependencies: +==================================== + +[1] Install nuget.exe somewhere on your system and add it to your PATH. + https://docs.nuget.org/consume/command-line-reference + https://dist.nuget.org/index.html + +[2] Download required nuget packages for third-party libraries. + Using a terminal window, type: + + make -C compat/vcbuild + + This will download the packages, unpack them into GEN.PKGS, + and populate the {include, lib, bin} directories in GEN.DEPS. + + +Building Git for Windows using VS2015: +====================================== + +[3] Build 64-bit version of Git for Windows. + Using a terminal window: + + make MSVC=1 DEBUG=1 + + +[4] Add compat/vcbuild/GEN.DEPS/bin to your PATH. + +[5] You should then be able to run the test suite and any interactive + commands. + +[6] To debug/profile in VS, open the git.exe in VS and run/debug + it. (Be sure to add GEN.DEPS/bin to the PATH in the debug + dialog.) + + +TODO List: +========== + +[A] config.mak.uname currently contains hard-coded paths + to the various MSVC and SDK libraries for the 64-bit + version of the compilers and libaries. + + See: SANE_TOOL_PATH, MSVC_DEPS, MSVC_SDK*, MSVC_VCDIR. + + Long term, we need to figure out how to properly import + values for %VCINSTALLDIR%, %LIB%, %LIBPATH%, and the + other values normally set by "vsvars32.bat" when a + developer command prompt is started. This would also + allow us to switch between 32- and 64-bit tool chains. + +[B] Currently, we leave the third-party DLLs we reference in + "compat/vcbuild/GEN.DEPS/bin". We need an installer + step to move them next to git.exe (or into libexec/git-core). + +[C] We need to build SLN or VCPROJ files. diff --git a/compat/vcbuild/nuget.config b/compat/vcbuild/nuget.config new file mode 100644 index 00000000000000..b5e734950478fa --- /dev/null +++ b/compat/vcbuild/nuget.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config new file mode 100644 index 00000000000000..8996f69840001a --- /dev/null +++ b/compat/vcbuild/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + From 657fbde25cb6e7bacdf8f9cd9101f574ae6ddb0f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 13:34:28 -0400 Subject: [PATCH 04/23] msvc: update compile helper for VS2015 Support -Z flags ("specify PDB options"), only include -l args on link commands, and force PDBs to be created. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a87d0da512e095..3bbfa47bf42cb8 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -12,10 +12,11 @@ use strict; my @args = (); my @cflags = (); +my @lflags = (); my $is_linking = 0; while (@ARGV) { my $arg = shift @ARGV; - if ("$arg" =~ /^-[DIMGO]/) { + if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { my $file_out = shift @ARGV; @@ -35,9 +36,11 @@ push(@args, "ssleay32.lib"); } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); + } elsif ("$arg" eq "-lexpat") { + push(@args, "libexpat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; - push(@args, $arg); + push(@lflags, $arg); } elsif ("$arg" =~ /^-R/) { # eat } else { @@ -45,6 +48,9 @@ } } if ($is_linking) { + push(@args, @lflags); + # force PDB to be created. + push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); From 7388541120938691bfced643a502c3be92a7c939 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 14:15:02 -0400 Subject: [PATCH 05/23] msvc: update Makefile and compiler settings for VS2015 Signed-off-by: Jeff Hostetler --- Makefile | 2 +- config.mak.uname | 47 ++++++++++++++++++++++++++++++++++++++++++----- git-compat-util.h | 9 +++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 2585b4d26e8781..dc3e24c0fc15e4 100644 --- a/Makefile +++ b/Makefile @@ -1008,7 +1008,7 @@ endif ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' diff --git a/config.mak.uname b/config.mak.uname index 4e43e83551f0fe..1e10ac01d5631d 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -339,6 +339,13 @@ endif ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; + # Prepend MSVC 64-bit tool-chain to PATH. + # + # A regular Git Bash *does not* have cl.exe in its $PATH. As there is a + # link.exe next to, and required by, cl.exe, we have to prepend this + # onto the existing $PATH. + # + SANE_TOOL_PATH ?= /c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64 HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -356,7 +363,13 @@ ifeq ($(uname_S),Windows) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease - SNPRINTF_RETURNS_BOGUS = YesPlease + # VS2015 with UCRT claims that snprintf and friends are C99 compliant, + # so we don't need this. + # + # TODO If we want to support older compilers, we need to make this + # TODO conditional on the compiler version. + # + # SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -378,22 +391,46 @@ ifeq ($(uname_S),Windows) CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl CFLAGS = - BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE + BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = + # Path to the unpacked third-party libraries + MSVC_DEPS = compat/vcbuild/GEN.DEPS + # Compensate for lack of %VCINSTALLDIR%, %LIB%, %LIBPATH%, and etc. + # since vcvars*.bat did not get a chance to setup the environment of + # the user's shell window. + # + # TODO If we ask the user to launch a "x64 Native" command prompt + # TODO and then have it start a git-bash window, these could be + # TODO inherited. So we wouldn't need to add these lines here. + # + MSVC_SDK81 = "c:/Program Files (x86)/Windows Kits/8.1" + MSVC_SDK10 = "c:/Program Files (x86)/Windows Kits/10" + MSVC_VCDIR = "c:/Program Files (x86)/Microsoft Visual Studio 14.0/VC" + BASIC_CFLAGS += \ + -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \ + -L$(MSVC_DEPS)/lib \ + -I$(MSVC_SDK81)/Include/um -I$(MSVC_SDK81)/Include/shared \ + -L$(MSVC_SDK81)/lib/winv6.3/um/x64 \ + -I$(MSVC_SDK10)/Include/10.0.10240.0/ucrt \ + -L$(MSVC_SDK10)/lib/10.0.10240.0/ucrt/x64 \ + -I$(MSVC_VCDIR)/INCLUDE \ + -L$(MSVC_VCDIR)/lib/amd64 + # Optionally enable memory leak reporting. + # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 ifndef DEBUG BASIC_CFLAGS += -GL -Os -MD BASIC_LDFLAGS += -LTCG AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd + BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG endif X = .exe diff --git a/git-compat-util.h b/git-compat-util.h index 80679a8b513195..fcf74f5bb7d748 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,15 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#ifdef USE_MSVC_CRTDBG +/* + * For these to work they must appear very early in each + * file -- before most of the standard header files. + */ +#include +#include +#endif + #define _FILE_OFFSET_BITS 64 From a29a15dba95791d89cdae1152816146120ebcd3a Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:07:20 +0100 Subject: [PATCH 06/23] msvc: include sigset_t definition On MSVC (VS2008) sigset_t is not defined. Signed-off-by: Philip Oakley --- compat/msvc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 580bb55bf4a71d..c64d5a56b20507 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,10 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +#ifdef _MSC_VER +typedef int sigset_t; +#endif + #include "compat/mingw.h" #endif From 906223035014bc71c521f049bdd5201ff62f021e Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:52:05 +0100 Subject: [PATCH 07/23] msvc: define O_ACCMODE This constant is not defined in MSVC's headers. In UCRT's fcntl.h, _O_RDONLY, _O_WRONLY and _O_RDWR are defined as 0, 1 and 2, respectively. Yes, that means that UCRT breaks with the tradition that O_RDWR == O_RDONLY | O_WRONLY. It is a perfectly legal way to define those constants, though, therefore we need to take care of defining O_ACCMODE accordingly. This is particularly important in order to keep our "open() can set errno to EISDIR" emulation working: it tests that (flags & O_ACCMODE) is not identical to O_RDONLY before going on to test specifically whether the file for which open() reported EACCES is, in fact, a directory. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/msvc.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index c64d5a56b20507..926e2fd5d6cf11 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -26,6 +26,9 @@ static __inline int strcasecmp (const char *s1, const char *s2) #ifdef _MSC_VER typedef int sigset_t; +/* open for reading, writing, or both (not in fcntl.h) */ +#define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) + #endif #include "compat/mingw.h" From 077e076d67e7663ffe8da8a40f180d12ce9ed014 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Apr 2016 10:48:13 +0100 Subject: [PATCH 08/23] msvc: fix the declaration of the _REPARSE_DATA_BUFFER structure GCC and MSVC disagree about using the GCC extension _ANONYMOUS_UNION. Simply skip that offending keyword when compiling with MSVC. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7039204e556fbb..edd95093aebde6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2287,7 +2287,10 @@ typedef struct _REPARSE_DATA_BUFFER { DWORD ReparseTag; WORD ReparseDataLength; WORD Reserved; - _ANONYMOUS_UNION union { +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { struct { WORD SubstituteNameOffset; WORD SubstituteNameLength; From e9fe955af2afea78d4046b86ce136ae9809e4c43 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 09/23] msvc: convert environment from/to UTF-16 on the fly This adds MSVC versions of getenv() and friends. These take UTF-8 arguments and return UTF-8 values, but use the UNICODE versions of the CRT routines. This avoids the need to write to __environ (which is only visible if you statically link to the CRT). This also avoids the CP_ACP conversions performed inside the CRT. It also avoids various memory leaks and problems. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 219 +++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 41 +++++++++ 2 files changed, 260 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index edd95093aebde6..fd81cff1646759 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1198,6 +1198,158 @@ static char *path_lookup(const char *cmd, char **path, int exe_only) return prog; } +#if defined(_MSC_VER) +/* + * Build an environment block combining the inherited environment + * merged with the given list of settings. + * + * Values of the form "KEY=VALUE" in deltaenv override inherited values. + * Values of the form "KEY" in deltaenv delete inherited values. + * + * We return a contiguous block of UNICODE strings with a final trailing + * zero word. + */ +static wchar_t *make_environment_block(char **deltaenv) +{ + /* + * The CRT (at least as of UCRT) secretly declares "_wenviron" + * as a function that returns a pointer to a mostly static table. + * Grab the pointer and cache it for the duration of our loop. + */ + extern wchar_t **_wenviron; + const wchar_t **my_wenviron = _wenviron; + + /* + * Internally, we create normal 'C' arrays of strings (pointing + * into the blocks) to help with some of the de-dup work. + */ + wchar_t **wptrs_ins = NULL; + wchar_t **wptrs_del = NULL; + wchar_t *wblock_ins = NULL; + wchar_t *wblock_del = NULL; + wchar_t *wend_ins; + wchar_t *wend_del; + wchar_t *w_ins; + wchar_t *w_del; + + int maxlen = 0; + int nr_delta = 0; + int nr_delta_ins = 0; + int nr_delta_del = 0; + int nr_wenv = 0; + int j, k, k_ins, k_del; + + /* + * Count the number of inserts and deletes in the deltaenv list. + * Allocate 'C' arrays for inserts and deletes. + * Also estimate the block size of our results. + */ + if (deltaenv && deltaenv[0] && *deltaenv[0]) { + for (k = 0; deltaenv && deltaenv[k] && *deltaenv[k]; k++) { + if (strchr(deltaenv[k], '=') == NULL) + nr_delta_del++; + else { + maxlen += strlen(deltaenv[k]) + 1; + nr_delta_ins++; + } + } + + if (nr_delta_ins) + wptrs_ins = (wchar_t**)calloc(nr_delta_ins + 1, sizeof(wchar_t*)); + if (nr_delta_del) + wptrs_del = (wchar_t**)calloc(nr_delta_del + 1, sizeof(wchar_t*)); + + nr_delta = nr_delta_ins + nr_delta_del; + } + while (my_wenviron && my_wenviron[nr_wenv] && *my_wenviron[nr_wenv]) + maxlen += wcslen(my_wenviron[nr_wenv++]) + 1; + maxlen++; + + /* + * Allocate blocks for inserted and deleted items. + * The individual pointers in the 'C' arrays will point into here. + * We will use the wblock_ins as the final result. + */ + if (nr_delta_del) { + wblock_del = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_del = wblock_del + maxlen; + w_del = wblock_del; + } + wblock_ins = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_ins = wblock_ins + maxlen; + w_ins = wblock_ins; + + /* + * deltaenv values override inherited environment, so put them + * in the result list first (so that we can de-dup using the + * wide versions of them. + * + * Items in the deltaenv list that DO NOT contain an "=" are + * treated as unsetenv. + * + * I'm going assume that there are no duplicates in deltaenv itself. + */ + k_ins = 0; + k_del = 0; + for (k = 0; k < nr_delta; k++) { + if (strchr(deltaenv[k], '=') == NULL) { + wptrs_del[k_del++] = w_del; + w_del += xutftowcs(w_del, deltaenv[k], (wend_del - w_del)); + *w_del++ = L'='; /* append '=' to make lookup easier in next step. */ + *w_del++ = 0; + } else { + wptrs_ins[k_ins++] = w_ins; + w_ins += xutftowcs(w_ins, deltaenv[k], (wend_ins - w_ins)) + 1; + } + } + assert(k_ins == nr_delta_ins); + assert(k_del == nr_delta_del); + + /* + * Walk the inherited environment and copy over unique, non-deleted + * ones into the result set. Note that we only have to de-dup WRT + * the values from deltaenv, because the inherited set should be unique. + */ + for (j = 0; j < nr_wenv; j++) { + const wchar_t *v_j = my_wenviron[j]; + wchar_t *v_j_eq = wcschr(v_j, L'='); + if (!v_j_eq) + continue; /* should not happen */ + int len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ + + /* lookup v_j in list of to-delete vars */ + for (k_del = 0; k_del < nr_delta_del; k_del++) { + if (wcsnicmp(v_j, wptrs_del[k_del], len_j_eq) == 0) + goto skip_it; + } + + /* lookup v_j in deltaenv portion of result set */ + for (k_ins = 0; k_ins < nr_delta_ins; k_ins++) { + if (wcsnicmp(v_j, wptrs_ins[k_ins], len_j_eq) == 0) + goto skip_it; + } + + /* item is unique, add it to results. */ + int len_j = wcslen(v_j); + memcpy(w_ins, v_j, len_j * sizeof(wchar_t)); + w_ins += len_j + 1; + +skip_it: + ; + } + + if (wptrs_ins) + free(wptrs_ins); + if (wptrs_del) + free(wptrs_del); + if (wblock_del) + free(wblock_del); + + return wblock_ins; +} + +#else + static int do_putenv(char **env, const char *name, int size, int free_old); /* used number of elements of environ array, including terminating NULL */ @@ -1245,6 +1397,8 @@ static wchar_t *make_environment_block(char **deltaenv) return wenvblk; } +#endif + struct pinfo_t { struct pinfo_t *next; pid_t pid; @@ -1503,6 +1657,69 @@ int mingw_kill(pid_t pid, int sig) return -1; } +#if defined(_MSC_VER) + +/* UTF8 versions of getenv and putenv (and unsetenv). + * Internally, they use the CRT's stock UNICODE routines + * to avoid data loss. + * + * Unlike the mingw version, we DO NOT directly write to + * the CRT variables. We also DO NOT try to manage/replace + * the CRT storage. + */ +char *msc_getenv(const char *name) +{ + int len_key, len_value; + wchar_t *w_key, *value; + const wchar_t *w_value; + + if (!name || !*name) + return NULL; + + len_key = strlen(name) + 1; + w_key = calloc(len_key, sizeof(wchar_t)); + xutftowcs(w_key, name, len_key); + w_value = _wgetenv(w_key); + free(w_key); + + if (!w_value) + return NULL; + + len_value = wcslen(w_value) * 3 + 1; + value = calloc(len_value, sizeof(char)); + xwcstoutf(value, w_value, len_value); + + /* TODO Warning: We return "value" which is an allocated + * value and the caller is NOT expecting to have to free + * it, so we leak memory. + */ + return value; +} + +int msc_putenv(const char *name) +{ + int len, result; + char *equal; + wchar_t *wide; + + if (!name || !*name) + return 0; + + len = strlen(name); + equal = strchr(name, '='); + wide = calloc(len+1+!equal, sizeof(wchar_t)); + xutftowcs(wide, name, len+1); + if (!equal) + wcscat(wide, L"="); + + result = _wputenv(wide); + + free(wide); + return result; +} + +#else + /* * Compare environment entries by key (i.e. stopping at '=' or '\0'). */ @@ -1631,6 +1848,8 @@ int mingw_putenv(const char *namevalue) return 0; } +#endif + /* * Note, this isn't a complete replacement for getaddrinfo. It assumes * that service contains a numerical port, or that it is null. It diff --git a/compat/mingw.h b/compat/mingw.h index 79dc3962ada068..520ea9791573af 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -259,12 +259,53 @@ char *mingw_getcwd(char *pointer, int len); #error "NO_UNSETENV is incompatible with the MinGW startup code!" #endif +#if defined(_MSC_VER) +/* + * We bind *env() routines (even the mingw_ ones) to private msc_ versions. + * These talk to the CRT using UNICODE/wchar_t, but maintain the original + * narrow-char API. + * + * Note that the MSCRT maintains both ANSI (getenv()) and UNICODE (_wgetenv()) + * routines and stores both versions of each environment variable in parallel + * (and secretly updates both when you set one or the other), but it uses CP_ACP + * to do the conversion rather than CP_UTF8. + * + * Since everything in the git code base is UTF8, we define the msc_ routines + * to access the CRT using the UNICODE routines and manually convert them to + * UTF8. This also avoids round-trip problems. + * + * This also helps with our linkage, since "_wenviron" is publicly exported + * from the CRT. But to access "_environ" we would have to statically link + * to the CRT (/MT). + * + * We also use "wmain(argc,argv,env)" and get the initial UNICODE setup for us. + * This avoids the need for the msc_startup() to import and convert the + * inherited environment. + * + * We require NO_SETENV (and let gitsetenv() call our msc_putenv). + */ +#define getenv msc_getenv +#define putenv msc_putenv +#define unsetenv msc_putenv +#define mingw_getenv msc_getenv +#define mingw_putenv msc_putenv +char *msc_getenv(const char *name); +int msc_putenv(const char *name); + +#ifndef NO_SETENV +#error "NO_SETENV is required for MSC startup code!" +#endif + +#else + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); #define putenv mingw_putenv #define unsetenv mingw_putenv +#endif + int mingw_gethostname(char *host, int namelen); #define gethostname mingw_gethostname From b9eaf91e326c16e7e556350420968987898de900 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 10/23] msvc: mark a variable as non-const VS2015 complains when using a const pointer in memcpy()/free(). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index fd81cff1646759..afb8f7035149af 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1581,7 +1581,10 @@ static int try_shell_exec(const char *cmd, char *const *argv) prog = path_lookup(interpr, path, 1); if (prog) { int argc = 0; - const char **argv2; +#ifndef _MSC_VER + const +#endif + char **argv2; while (argv[argc]) argc++; ALLOC_ARRAY(argv2, argc + 1); argv2[0] = (char *)cmd; /* full path to the script file */ From 07db451564c5df122d85a5d08c03d1be26629abb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 11/23] msvc: do not pretend to support all signals This special-cases various signals that are not supported on Windows, such as SIGPIPE. These cause the UCRT to throw asserts (at least in debug mode). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index afb8f7035149af..6ddd806b36755d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2410,8 +2410,34 @@ int mingw_raise(int sig) sigint_fn(SIGINT); return 0; +#if defined(_MSC_VER) + /* + * in the CRT defines 8 signals as being + * supported on the platform. Anything else causes + * an "Invalid signal or error" (which in DEBUG builds + * causes the Abort/Retry/Ignore dialog). We by-pass + * the CRT for things we already know will fail. + */ + /*case SIGINT:*/ + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + case SIGABRT_COMPAT: + return raise(sig); + default: + errno = EINVAL; + return -1; + +#else + default: return raise(sig); + +#endif + } } From 1e5cef4c5da98d49aa36cd0180dbcea64e6ad1ee Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 12/23] msvc: provide a main() wrapper similar to mingw_main() The MINGW version of the main() wrapper gets away with declaring symbols that were intentionally not exported. However, some of these symbols do not actually exist in MSVC's UCRT. So let's add an MSVC version of the main() wrapper that uses wmain() and imports the UNICODE argv and environment. While at it, we pass our UTF-8 version of ARGV to the real main -- rather than overwriting __argv as is done in the MINGW Version. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 20 ++++++++++++ 2 files changed, 109 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 6ddd806b36755d..7536c3acb241e3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2916,6 +2916,7 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +#if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2928,6 +2929,7 @@ typedef struct { extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob, _startupinfo *si); +#endif static NORETURN void die_startup(void) { @@ -2949,6 +2951,91 @@ static char *wcstoutfdup_startup(char *buffer, const wchar_t *wcs, size_t len) return memcpy(malloc_startup(len), buffer, len); } +#if defined(_MSC_VER) + +/* + * This routine sits between wmain() and "main" in git.exe. + * We receive UNICODE (wchar_t) values for argv and env. + * + * To be more compatible with the core git code, we convert + * argv into UTF8 and pass them directly to the "main" routine. + * + * We don't bother converting the given UNICODE env vector, + * but rather leave them in the CRT. We replaced the various + * getenv/putenv routines to pull them directly from the CRT. + * + * This is unlike the MINGW version: + * [] It does the UNICODE-2-UTF8 conversion on both sets and + * stuffs the values back into the CRT using exported symbols. + * [] It also maintains a private copy of the environment and + * tries to track external changes to it. + */ +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env) +{ + char **my_utf8_argv = NULL, **save = NULL; + char *buffer = NULL; + int maxlen; + int k, x; + +#ifdef USE_MSVC_CRTDBG + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif + + /* determine size of argv conversion buffer */ + maxlen = wcslen(_wpgmptr); + for (k = 1; k < argc; k++) + maxlen = max(maxlen, wcslen(w_argv[k])); + + /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ + maxlen = 3 * maxlen + 1; + buffer = malloc_startup(maxlen); + + /* + * Create a UTF-8 version of w_argv. Also create a "save" copy + * to remember all the string pointers because parse_options() + * will remove claimed items from the argv that we pass down. + */ + ALLOC_ARRAY(my_utf8_argv, argc + 1); + ALLOC_ARRAY(save, argc + 1); + save[0] = my_utf8_argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); + for (k = 1; k < argc; k++) + save[k] = my_utf8_argv[k] = wcstoutfdup_startup(buffer, w_argv[k], maxlen); + save[k] = my_utf8_argv[k] = NULL; + + free(buffer); + + /* fix Windows specific environment settings */ + setup_windows_environment(); + + /* initialize critical section for waitpid pinfo_t list */ + InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); + + /* set up default file mode and file modes for stdin/out/err */ + _fmode = _O_BINARY; + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + _setmode(_fileno(stderr), _O_BINARY); + + /* initialize Unicode console */ + winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + + /* invoke the real main() using our utf8 version of argv. */ + int exit_status = msc_main(argc, my_utf8_argv); + + for (k = 0; k < argc; k++) + free(save[k]); + free(save); + free(my_utf8_argv); + + return exit_status; +} + +#else + void mingw_startup(void) { int i, maxlen, argc; @@ -3024,6 +3111,8 @@ void mingw_startup(void) current_directory_len = GetCurrentDirectoryW(0, NULL); } +#endif + int uname(struct utsname *buf) { unsigned v = (unsigned)GetVersion(); diff --git a/compat/mingw.h b/compat/mingw.h index 520ea9791573af..fdc41749e97dac 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -686,7 +686,25 @@ extern CRITICAL_SECTION pinfo_cs; /* * A replacement of main() that adds win32 specific initialization. + * + * Note that the end of these macros are unterminated so that the + * brace group following the use of the macro is the body of the + * function. */ +#if defined(_MSC_VER) + +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env); + +#define main(c,v) dummy_decl_msc_main(void); \ +int wmain(int my_argc, \ + wchar_t **my_w_argv, \ + wchar_t **my_w_env) \ +{ \ + return msc_startup(my_argc, my_w_argv, my_w_env); \ +} \ +int msc_main(c, v) + +#else void mingw_startup(void); #define main(c,v) dummy_decl_mingw_main(void); \ @@ -698,6 +716,8 @@ int main(int argc, const char **argv) \ } \ static int mingw_main(c,v) +#endif + /* * Used by Pthread API implementation for Windows */ From 8a1aac480db93c0c6b6756ff73f1edbfc1f6579e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 13/23] msvc: do not re-declare the timespec struct VS2015's headers already declare that struct. Signed-off-by: Jeff Hostetler --- compat/mingw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index fdc41749e97dac..8df25d543b1df7 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -388,11 +388,13 @@ static inline long long filetime_to_hnsec(const FILETIME *ft) #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#ifndef _MSC_VER struct timespec { time_t tv_sec; long tv_nsec; }; #endif +#endif static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) { From 69b6ea107b7a12db608d5f3bc815439e6373dccd Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 14/23] msvc: define ftello() It is just called different in MSVC's headers. Signed-off-by: Jeff Hostetler --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 926e2fd5d6cf11..9a3a9269816bb6 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -25,6 +25,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR #ifdef _MSC_VER +#define ftello _ftelli64 + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) From b9a0f8e2681097fb37b7b1de4dec1b51c02c1df6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 15/23] msvc: fix isatty() The hack that works in MINGW does not work with MSVC's CRT. Add MSVC versions of isatty() and swap_osfhnd(). The MINGW versions attempt to replace the underlying OS HANDLE in an existing file descriptor (fd) by writing to some undocumented fields in the "ioinfo" structures inside the CRT. These structures changed size and shape with the new UCRT in VS2015. The new MSVC versions of these routines work without touching private fields. In theory, we should be able to replace the ming versions with this one. Signed-off-by: Jeff Hostetler --- compat/msvc.h | 3 ++ compat/winansi.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 9a3a9269816bb6..1b6e525f2f9cae 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -27,6 +27,9 @@ static __inline int strcasecmp (const char *s1, const char *s2) #ifdef _MSC_VER #define ftello _ftelli64 +#define isatty msc_isatty +int msc_isatty(int); + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) diff --git a/compat/winansi.c b/compat/winansi.c index a3b7208fd8f00a..c8e12b0e2ad3ca 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -8,6 +8,22 @@ #include #include "win32.h" +#if defined(_MSC_VER) + +static int fd_is_interactive[3] = { 0, 0, 0 }; +#define MY_INTERACTIVE_CONSOLE 0x1 +#define MY_INTERACTIVE_SWAPPED 0x2 +#define MY_INTERACTIVE_MSYS 0x4 + +/* Accumulate what we know about the inherited console descriptors. */ +static void set_interactive(int fd, int bit) +{ + if (fd >=0 && fd <= 2) + fd_is_interactive[fd] |= bit; +} + +#endif + /* ANSI codes used by git: m, K @@ -94,6 +110,10 @@ static int is_console(int fd) if (!GetConsoleScreenBufferInfo(hcon, &sbi)) return 0; +#if defined(_MSC_VER) + set_interactive(fd, MY_INTERACTIVE_CONSOLE); +#endif + /* initialize attributes */ if (!initialized) { console = hcon; @@ -455,6 +475,55 @@ static HANDLE duplicate_handle(HANDLE hnd) return hresult; } +#if defined(_MSC_VER) +static HANDLE swap_osfhnd(int fd, HANDLE new_handle) +{ + assert((fd == 1) || (fd == 2)); + DWORD key_std = ((fd == 1) ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE); + + /* + * Create a copy of the original handle associated with fd + * because the original will get closed when we dup2(). + */ + HANDLE h_original = (HANDLE)_get_osfhandle(fd); + HANDLE h_copy_original = duplicate_handle(h_original); + + /* Create a temp fd associated with the already open "new_handle". */ + int fd_temp = _open_osfhandle((intptr_t)new_handle, O_BINARY); + + /* + * Use stock dup2() to re-bind fd to the new handle. Note that + * this will implicitly close(1) and close both fd=1 and the + * originally associated handle. It will open a new fd=1 and + * call DuplicateHandle() on the handle associated with fd_temp. + * It is because of this implicit close() that we created the + * copy of the original. + * + * Note that the OS can recycle HANDLE (numbers) just like it + * recycles fd (numbers), so we must update the cached value + * of "console". You can use GetFileType() to see that + * h_original and _get_osfhandle(fd) may have the same number + * value, but they refer to different actual files now. + * + * Note that dup2() when given target := {0,1,2} will also + * call SetStdHandle(), so we don't need to worry about that. + */ + dup2(fd_temp, fd); + if (console == h_original) + console = h_copy_original; + h_original = INVALID_HANDLE_VALUE; + + /* Close the temp fd. This explicitly closes "new_handle" + * (because it has been associated with it). + */ + close(fd_temp); + + fd_is_interactive[fd] |= MY_INTERACTIVE_SWAPPED; + + return h_copy_original; +} + +#else /* * Make MSVCRT's internal file descriptor control structure accessible @@ -527,10 +596,26 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) return old_handle; } +#endif + + #ifdef DETECT_MSYS_TTY #include + +#if defined(_MSC_VER) + +typedef struct _OBJECT_NAME_INFORMATION +{ + UNICODE_STRING Name; + WCHAR NameBuffer[0]; +} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION; + +#define ObjectNameInformation 1 + +#else #include +#endif static void detect_msys_tty(int fd) { @@ -559,6 +644,9 @@ static void detect_msys_tty(int fd) !wcsstr(name, L"-pty")) return; +#if defined(_MSC_VER) + fd_is_interactive[fd] |= MY_INTERACTIVE_MSYS; +#else /* init ioinfo size if we haven't done so */ if (init_sizeof_ioinfo()) return; @@ -566,6 +654,7 @@ static void detect_msys_tty(int fd) /* set FDEV flag, reset FPIPE flag */ _pioinfo(fd)->osflags &= ~FPIPE; _pioinfo(fd)->osflags |= FDEV; +#endif } #endif @@ -578,6 +667,12 @@ void winansi_init(void) /* check if either stdout or stderr is a console output screen buffer */ con1 = is_console(1); con2 = is_console(2); + +#if defined(_MSC_VER) + /* Also compute console bit for fd 0 even though we don't need the result here. */ + is_console(0); +#endif + if (!con1 && !con2) { #ifdef DETECT_MSYS_TTY /* check if stdin / stdout / stderr are MSYS2 pty pipes */ @@ -630,3 +725,23 @@ HANDLE winansi_get_osfhandle(int fd) } return hnd; } + +#ifdef _MSC_VER + +/* Wrapper for isatty(). Most calls in the main git code + * call isatty(1 or 2) to see if the instance is interactive + * and should: be colored, show progress, paginate output. + * We lie and give results for what the descriptor WAS at + * startup (and ignore any pipe redirection we internally + * do). + */ +#undef isatty +int msc_isatty(fd) +{ + if (fd >=0 && fd <= 2) + return fd_is_interactive[fd] != 0; + else + return isatty(fd); +} + +#endif From 46d84bb04e8f984e0df63b600c4c681af1e0feaf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 16:01:35 -0400 Subject: [PATCH 16/23] cache-tree.c: avoid reusing the DEBUG constant In MSVC, the DEBUG constant is set automatically whenever compiling with debug information. This is clearly not what was intended in cache-tree.c, so let's use a less ambiguous constant there. Signed-off-by: Jeff Hostetler --- cache-tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index f28b1f45a49b84..5ff261f9c9cab1 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -4,8 +4,8 @@ #include "tree-walk.h" #include "cache-tree.h" -#ifndef DEBUG -#define DEBUG 0 +#ifndef DEBUG_CACHE_TREE +#define DEBUG_CACHE_TREE 0 #endif struct cache_tree *cache_tree(void) @@ -110,7 +110,7 @@ static int do_invalidate_path(struct cache_tree *it, const char *path) int namelen; struct cache_tree_sub *down; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree invalidate <%s>\n", path); #endif @@ -392,7 +392,7 @@ static int update_one(struct cache_tree *it, strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); strbuf_add(&buffer, sha1, 20); -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one %o %.*s\n", mode, entlen, path + baselen); #endif @@ -414,7 +414,7 @@ static int update_one(struct cache_tree *it, strbuf_release(&buffer); it->entry_count = to_invalidate ? -1 : i - *skip_count; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one (%d ent, %d subtree) %s\n", it->entry_count, it->subtree_nr, sha1_to_hex(it->sha1)); @@ -453,7 +453,7 @@ static void write_one(struct strbuf *buffer, struct cache_tree *it, strbuf_add(buffer, path, pathlen); strbuf_addf(buffer, "%c%d %d\n", 0, it->entry_count, it->subtree_nr); -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n", pathlen, path, it->entry_count, it->subtree_nr, @@ -526,7 +526,7 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) size -= 20; } -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%s> (%d ent, %d subtree) %s\n", *buffer, it->entry_count, subtree_nr, From 4f0579e160b01d576c1461b405e4ac769303d02d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 6 Jun 2016 11:06:28 -0400 Subject: [PATCH 17/23] msvc: fix setvbuf() call The VS2015 version of the CRT asserts when you pass a zero buffer length and request line buffering. This fix sets it to the default BUFSIZ. Signed-off-by: Jeff Hostetler --- usage.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usage.c b/usage.c index 0efa3faf601231..988695ea87a4a6 100644 --- a/usage.c +++ b/usage.c @@ -15,7 +15,12 @@ void vreportf(const char *prefix, const char *err, va_list params) fflush(fh); if (!tweaked_error_buffering) { +#if defined(_MSC_VER) + /* UCRT doesn't like zero buffer size */ + setvbuf(fh, NULL, _IOLBF, BUFSIZ); +#else setvbuf(fh, NULL, _IOLBF, 0); +#endif tweaked_error_buffering = 1; } From ace8bd636cce193babfa60d7d0e0b09a308672f2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 9 Jun 2016 09:40:13 -0400 Subject: [PATCH 18/23] msvc: release mode PDBs and library DLLs Install required third-party DLLs next to EXEs. Build and install release mode PDBs for git executables allowing detailed stack traces in the event of crash. Signed-off-by: Jeff Hostetler --- Makefile | 22 ++++++++++++++++++++++ compat/vcbuild/Makefile | 3 ++- compat/vcbuild/scripts/clink.pl | 7 ++++--- config.mak.uname | 10 +++++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index dc3e24c0fc15e4..67b4a35b49c799 100644 --- a/Makefile +++ b/Makefile @@ -2353,6 +2353,28 @@ install: all $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index b94bcd119a569b..cfca611dfcb3b8 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -106,13 +106,14 @@ openssl: insdir ZLIB_VER=1.2.8.8 ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) ZLIB_INC=$(ZLIB_ROOT)/build/native/include -ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/RelWithDebInfo zlib: insdir [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib cp -r $(ZLIB_INC)/* $(INST_INC)/zlib cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + cp -r $(ZLIB_LIB)/*.pdb $(INST_BIN)/ ################################################################## clean: diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bbfa47bf42cb8..a8df6bb63b07c2 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -22,9 +22,12 @@ my $file_out = shift @ARGV; if ("$file_out" =~ /exe$/) { $is_linking = 1; + # Create foo.exe and foo.pdb push(@args, "-OUT:$file_out"); } else { + # Create foo.o and foo.o.pdb push(@args, "-Fo$file_out"); + push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { push(@args, "zlib.lib"); @@ -49,12 +52,10 @@ } if ($is_linking) { push(@args, @lflags); - # force PDB to be created. - push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); push(@args, @cflags); } -#printf("**** @args\n"); +printf("**** @args\n\n\n"); exit (system(@args) != 0); diff --git a/config.mak.uname b/config.mak.uname index 1e10ac01d5631d..8f96a6e6da996f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -425,12 +425,16 @@ ifeq ($(uname_S),Windows) # Optionally enable memory leak reporting. # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + # Always give "-Zi" to the compiler and "-debug" to linker (even in + # release mode) to force a PDB to be generated (like RelWithDebInfo). + BASIC_CFLAGS += -Zi + BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Os -MD - BASIC_LDFLAGS += -LTCG + BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG + BASIC_CFLAGS += -MDd -DDEBUG -D_DEBUG endif X = .exe From 1af1417f5673e52a582cb241f051503a19163812 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 14 Oct 2016 06:01:39 -0700 Subject: [PATCH 19/23] msvc: respect the quiet-by-default output Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a8df6bb63b07c2..b67b8313043352 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -57,5 +57,5 @@ unshift(@args, "cl.exe"); push(@args, @cflags); } -printf("**** @args\n\n\n"); +printf(STDERR "**** @args\n\n\n") if (!defined($ENV{'QUIET_GEN'})); exit (system(@args) != 0); From 026e29f2808b3ef2b07c1161ba80b4792c4c4da4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 Oct 2016 14:21:42 -0400 Subject: [PATCH 20/23] msvc: use OpenSSL's SHA-1 routines Just like 1e2ce1d (sha1: Use OpenSSL SHA1 routines on MINGW, 2016-10-12), we now use OpenSSL's SHA-1 routines instead of Git's own because OpenSSL is substantially faster as of version 1.0.2: it now uses hardware acceleration on Intel processors much more effectively. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 8f96a6e6da996f..20d30b935f2669 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -380,7 +380,6 @@ ifeq ($(uname_S),Windows) NO_REGEX = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_INET_PTON = YesPlease NO_INET_NTOP = YesPlease From b24dff6b65e03e758eaaaad8326b6d3aee4a3fc0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 08:21:05 -0700 Subject: [PATCH 21/23] msvc: use libiconv Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 22 +++++++++++++++++++++- compat/vcbuild/packages.config | 3 +++ config.mak.uname | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index cfca611dfcb3b8..a9d1a6b92bef82 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -9,7 +9,8 @@ INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS ################################################################## -all: unpack expat libssh libssh_redist curl curl_redist openssl zlib +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ + libiconv libiconv_redist unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) @@ -38,6 +39,25 @@ expat: insdir cp -r $(EXPAT_LIB)/* $(INST_LIB)/ +################################################################## + +LIBICONV_VER=1.14.0.11 +LIBICONV_ROOT=$(PKGDIR)/libiconv.$(LIBICONV_VER)/build/native +LIBICONV_INC=$(LIBICONV_ROOT)/include +LIBICONV_LIB=$(LIBICONV_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libiconv: insdir + cp -r $(LIBICONV_INC)/* $(INST_INC)/ + cp -r $(LIBICONV_LIB)/libiconv.lib $(INST_LIB)/iconv.lib + + +LIBICONV_REDIST_ROOT=$(PKGDIR)/libiconv.redist.$(LIBICONV_VER)/build/native +LIBICONV_REDIST_BIN=$(LIBICONV_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libiconv_redist: insdir + cp -r $(LIBICONV_REDIST_BIN)/* $(INST_BIN)/ + + ################################################################## LIBSSH_VER=1.4.3.3 diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config index 8996f69840001a..5598c7b7d0bbd7 100644 --- a/compat/vcbuild/packages.config +++ b/compat/vcbuild/packages.config @@ -5,6 +5,9 @@ + + + diff --git a/config.mak.uname b/config.mak.uname index 20d30b935f2669..640b196968cc7a 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -358,8 +358,7 @@ ifeq ($(uname_S),Windows) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease NO_MEMMEM = YesPlease - # NEEDS_LIBICONV = YesPlease - NO_ICONV = YesPlease + NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease From be11f2da2c4b1c4b78cc4c4feaed846bb7b57895 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Oct 2016 02:46:59 -0700 Subject: [PATCH 22/23] msvc: work around iconv() not setting errno When compiling with MSVC, we rely on NuPkgs to provide the binaries of dependencies such as libiconv. The libiconv 1.14.0.11 package available from https://www.nuget.org/packages/libiconv seems to have a bug where it does not set errno (when we would expect it to be E2BIG). Let's simulate the error condition by taking less than 16 bytes remaining in the out buffer as an indicator that we ran out of space. While 16 might seem a bit excessive (when converting from, say, any encoding to UTF-8, 8 bytes should be fine), it is designed to be a safe margin. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/git-compat-util.h b/git-compat-util.h index fcf74f5bb7d748..75a5f19c9c2b6a 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -278,6 +278,32 @@ extern char *gitdirname(char *); #ifndef NO_ICONV #include +#ifdef _MSC_VER +/* + * At least version 1.14.0.11 of the libiconv NuPkg at + * https://www.nuget.org/packages/libiconv/ does not set errno at all. + * + * Let's simulate it by testing whether we might have possibly run out of + * space. + */ +static inline size_t msvc_iconv(iconv_t conv, + const char **inpos, size_t *insize, + char **outpos, size_t *outsize) +{ + int saved_errno = errno, res; + + errno = ENOENT; + res = iconv(conv, inpos, insize, outpos, outsize); + if (!res) + errno = saved_errno; + else if (errno == ENOENT) + errno = *outsize < 16 ? E2BIG : 0; + + return res; +} +#undef iconv +#define iconv msvc_iconv +#endif #endif #ifndef NO_OPENSSL From c0abebad2c3257b894476689f5741f08b2c21ca0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 23 Oct 2016 01:22:20 -0700 Subject: [PATCH 23/23] t7800: fix quoting When passing a command-line to call an external diff command to the difftool, we must be prepared for paths containing special characters, e.g. backslashes in the temporary directory's path on Windows. This has been caught by running the test suite with an MSVC-built Git: in contrast to the MINGW one, it does not rewrite `$TMP` to use forward slashes instead of backslashes. Signed-off-by: Johannes Schindelin --- t/t7800-difftool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 70a2de461af581..e0f81bdb220910 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -320,7 +320,7 @@ test_expect_success PERL 'difftool --extcmd cat arg1' ' test_expect_success PERL 'difftool --extcmd cat arg2' ' echo branch >expect && git difftool --no-prompt \ - --extcmd sh\ -c\ \"cat\ \$2\" branch >actual && + --extcmd sh\ -c\ \"cat\ \\\"\$2\\\"\" branch >actual && test_cmp expect actual '