Merge pull request #371 from dscho/run-scalar-functional-tests-and-fix-built-in-fsmonitor

Fix the built-in FSMonitor, and run Scalar's Functional Tests as part of the automated builds

Author: dscho

.github/workflows/scalar-functional-tests.yml

@@ -0,0 +1,220 @@
+name: Scalar Functional Tests
+
+env:
+  SCALAR_REPOSITORY: microsoft/scalar
+  SCALAR_REF: main
+  DEBUG_WITH_TMATE: false
+  SCALAR_TEST_SKIP_VSTS_INFO: true
+
+on:
+  push:
+    branches: [ vfs-*, tentative/vfs-* ]
+  pull_request:
+    branches: [ vfs-*, features/* ]
+
+jobs:
+  scalar:
+    name: "Scalar Functional Tests"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Order by runtime (in descending order)
+        os: [windows-2019, macos-11, ubuntu-20.04, ubuntu-22.04]
+        # Scalar.NET used to be tested using `features: [false, experimental]`
+        # But currently, Scalar/C ignores `feature.scalar` altogether, so let's
+        # save some electrons and run only one of them...
+        features: [ignored]
+        exclude:
+          # The built-in FSMonitor is not (yet) supported on Linux
+          - os: ubuntu-20.04
+            features: experimental
+          - os: ubuntu-22.04
+            features: experimental
+    runs-on: ${{ matrix.os }}
+
+    env:
+      BUILD_FRAGMENT: bin/Release/netcoreapp3.1
+      GIT_FORCE_UNTRACKED_CACHE: 1
+
+    steps:
+      - name: Check out Git's source code
+        uses: actions/checkout@v3
+
+      - name: Setup build tools on Windows
+        if: runner.os == 'Windows'
+        uses: git-for-windows/setup-git-for-windows-sdk@v1
+
+      - name: Provide a minimal `install` on Windows
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          test -x /usr/bin/install ||
+          tr % '\t' >/usr/bin/install <<-\EOF
+          #!/bin/sh
+
+          cmd=cp
+          while test $# != 0
+          do
+          %case "$1" in
+          %-d) cmd="mkdir -p";;
+          %-m) shift;; # ignore mode
+          %*) break;;
+          %esac
+          %shift
+          done
+
+          exec $cmd "$@"
+          EOF
+
+      - name: Install build dependencies for Git (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
+
+      - name: Build and install Git
+        shell: bash
+        env:
+          NO_TCLTK: Yup
+        run: |
+          # We do require a VFS version
+          def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
+          test -n "$def_ver"
+
+          # Ensure that `git version` reflects DEF_VER
+          case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
+          ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
+          *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
+          esac
+
+          SUDO=
+          extra=
+          case "${{ runner.os }}" in
+          Windows)
+            extra=DESTDIR=/c/Progra~1/Git
+            cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
+            ;;
+          Linux)
+            SUDO=sudo
+            extra=prefix=/usr
+            ;;
+          macOS)
+            SUDO=sudo
+            extra=prefix=/usr/local
+            ;;
+          esac
+
+          $SUDO make -j5 $extra install
+
+      - name: Ensure that we use the built Git and Scalar
+        shell: bash
+        run: |
+          type -p git
+          git version
+          case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
+          type -p scalar
+          scalar version
+          case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
+
+      - name: Check out Scalar's source code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
+          path: scalar
+          repository: ${{ env.SCALAR_REPOSITORY }}
+          ref: ${{ env.SCALAR_REF }}
+
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '3.1.x'
+
+      - name: Install dependencies
+        run: dotnet restore
+        working-directory: scalar
+        env:
+          DOTNET_NOLOGO: 1
+
+      - name: Build
+        working-directory: scalar
+        run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.
+
+      - name: Setup platform (Linux)
+        if: runner.os == 'Linux'
+        run: |
+          echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
+          echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+      - name: Setup platform (Mac)
+        if: runner.os == 'macOS'
+        run: |
+          echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
+          echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+      - name: Setup platform (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
+          echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
+          echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
+
+      - name: Configure feature.scalar
+        run: git config --global feature.scalar ${{ matrix.features }}
+
+      - id: functional_test
+        name: Functional test
+        timeout-minutes: 60
+        working-directory: scalar
+        shell: bash
+        run: |
+          export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
+          export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
+          export GIT_TRACE2_EVENT_BRIEF=true
+          export GIT_TRACE2_PERF_BRIEF=true
+          mkdir -p "$TRACE2_BASENAME"
+          mkdir -p "$TRACE2_BASENAME/Event"
+          mkdir -p "$TRACE2_BASENAME/Perf"
+          git version --build-options
+          cd ../out
+          Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
+
+      - name: Force-stop FSMonitor daemons and Git processes (Windows)
+        if: runner.os == 'Windows' && (success() || failure())
+        shell: bash
+        run: |
+          set -x
+          wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
+          wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
+          wmic process where "ExecutablePath Like '%git.exe'" delete
+
+      - id: trace2_zip_unix
+        if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+        name: Zip Trace2 Logs (Unix)
+        shell: bash
+        working-directory: scalar
+        run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/
+
+      - id: trace2_zip_windows
+        if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+        name: Zip Trace2 Logs (Windows)
+        working-directory: scalar
+        run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}
+
+      - name: Archive Trace2 Logs
+        if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.TRACE2_BASENAME }}.zip
+          path: scalar/${{ env.TRACE2_BASENAME }}.zip
+          retention-days: 3
+
+      # The GitHub Action `action-tmate` allows developers to connect to the running agent
+      # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
+      # environment in `C:\msys64`, therefore it can be slightly tricky to interact with
+      # Git for Windows, which runs a slightly incompatible MSYS2 runtime).
+      - name: action-tmate
+        if: env.DEBUG_WITH_TMATE == 'true' && failure()
+        uses: mxschmitt/action-tmate@v3
+        with:
+          limit-access-to-actor: true

Documentation/config/core.txt

@@ -806,3 +806,12 @@ core.abbrev::
 	If set to "no", no abbreviation is made and the object names
 	are shown in their full length.
 	The minimum length is 4.
+
+core.configWriteLockTimeoutMS::
+	When processes try to write to the config concurrently, it is likely
+	that one process "wins" and the other process(es) fail to lock the
+	config file. By configuring a timeout larger than zero, Git can be
+	told to try to lock the config again a couple times within the
+	specified timeout. If the timeout is configure to zero (which is the
+	default), Git will fail immediately when the config is already
+	locked.

Documentation/scalar.txt

@@ -8,14 +8,17 @@ scalar - A tool for managing large Git repositories
 SYNOPSIS
 --------
 [verse]
-scalar clone [--single-branch] [--branch <main-branch>] [--full-clone] <url> [<enlistment>]
+scalar clone [--single-branch] [--branch <main-branch>] [--full-clone]
+	[--local-cache-path <path>] [--cache-server-url <url>] [--[no-]src]
+	<url> [<enlistment>]
 scalar list
 scalar register [<enlistment>]
 scalar unregister [<enlistment>]
 scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files ) [<enlistment>]
 scalar reconfigure [ --all | <enlistment> ]
 scalar diagnose [<enlistment>]
 scalar delete <enlistment>
+scalar cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]
 
 DESCRIPTION
 -----------
@@ -80,10 +83,24 @@ remote-tracking branch for the branch this option was used for the initial
 cloning. If the HEAD at the remote did not point at any branch when
 `--single-branch` clone was made, no remote-tracking branch is created.
 
+--no-src::
+	Skip adding a `src` directory within the target enlistment.
+
 --[no-]full-clone::
 	A sparse-checkout is initialized by default. This behavior can be
 	turned off via `--full-clone`.
 
+--local-cache-path <path>::
+    Override the path to the local cache root directory; Pre-fetched objects
+    are stored into a repository-dependent subdirectory of that path.
++
+The default is `<drive>:\.scalarCache` on Windows (on the same drive as the
+clone), and `~/.scalarCache` on macOS.
+
+--cache-server-url <url>::
+    Retrieve missing objects from the specified remote, which is expected to
+    understand the GVFS protocol.
+
 List
 ~~~~
 
@@ -157,6 +174,27 @@ delete <enlistment>::
 	This subcommand lets you delete an existing Scalar enlistment from your
 	local file system, unregistering the repository.
 
+Cache-server
+~~~~~~~~~~~~
+
+cache-server ( --get | --set <url> | --list [<remote>] ) [<enlistment>]::
+    This command lets you query or set the GVFS-enabled cache server used
+    to fetch missing objects.
+
+--get::
+    This is the default command mode: query the currently-configured cache
+    server URL, if any.
+
+--list::
+    Access the `gvfs/info` endpoint of the specified remote (default:
+    `origin`) to figure out which cache servers are available, if any.
++
+In contrast to the `--get` command mode (which only accesses the local
+repository), this command mode triggers a request via the network that
+potentially requires authentication. If authentication is required, the
+configured credential helper is employed (see linkgit:git-credential[1]
+for details).
+
 SEE ALSO
 --------
 linkgit:git-clone[1], linkgit:git-maintenance[1].

Makefile

@@ -2736,6 +2736,7 @@ GIT_OBJS += git.o
 .PHONY: git-objs
 git-objs: $(GIT_OBJS)
 
+SCALAR_OBJS := json-parser.o
 SCALAR_OBJS += scalar.o
 .PHONY: scalar-objs
 scalar-objs: $(SCALAR_OBJS)
@@ -2884,7 +2885,7 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
 		$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
 
-scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
+scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
 	$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
 		$(filter %.o,$^) $(LIBS)
 

abspath.c

@@ -14,7 +14,7 @@ int is_directory(const char *path)
 }
 
 /* removes the last path component from 'path' except if 'path' is root */
-static void strip_last_component(struct strbuf *path)
+void strip_last_path_component(struct strbuf *path)
 {
 	size_t offset = offset_1st_component(path->buf);
 	size_t len = path->len;
@@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
 			continue; /* '.' component */
 		} else if (next.len == 2 && !strcmp(next.buf, "..")) {
 			/* '..' component; strip the last path component */
-			strip_last_component(resolved);
+			strip_last_path_component(resolved);
 			continue;
 		}
 
@@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
 				 * strip off the last component since it will
 				 * be replaced with the contents of the symlink
 				 */
-				strip_last_component(resolved);
+				strip_last_path_component(resolved);
 			}
 
 			/*

abspath.h

@@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error);
 const char *absolute_path(const char *path);
 char *absolute_pathdup(const char *path);
 
+/**
+ * Remove the last path component from 'path' except if 'path' is root.
+ */
+void strip_last_path_component(struct strbuf *path);
+
 /*
  * Concatenate "prefix" (if len is non-zero) and "path", with no
  * connecting characters (so "prefix" should end with a "/").

config.c

@@ -3465,6 +3465,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
 					   const char *value_pattern,
 					   unsigned flags)
 {
+	static unsigned long timeout_ms = ULONG_MAX;
 	int fd = -1, in_fd = -1;
 	int ret;
 	struct lock_file lock = LOCK_INIT;
@@ -3485,11 +3486,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
 	if (!config_filename)
 		config_filename = filename_buf = git_pathdup("config");
 
+	if ((long)timeout_ms < 0 &&
+	    git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
+		timeout_ms = 0;
+
 	/*
 	 * The lock serves a purpose in addition to locking: the new
 	 * contents of .git/config will be written into it.
 	 */
-	fd = hold_lock_file_for_update(&lock, config_filename, 0);
+	fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
+					       timeout_ms);
 	if (fd < 0) {
 		error_errno(_("could not lock config file %s"), config_filename);
 		ret = CONFIG_NO_LOCK;

contrib/buildsystems/CMakeLists.txt

@@ -797,7 +797,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main)
 add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c)
 target_link_libraries(git-shell common-main)
 
-add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c)
+add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c)
 target_link_libraries(scalar common-main)
 
 if(CURL_FOUND)