maintenance: add cache-local-objects maintenance task

Introduce a new maintenance task, `cache-local-objects`, that operates
on Scalar or VFS for Git repositories with a per-volume, shared object
cache (specified by `gvfs.sharedCache`) to migrate packfiles and loose
objects from the repository object directory to the shared cache.

Older versions of `microsoft/git` incorrectly placed packfiles in the
repository object directory instead of the shared cache; this task will
help clean up existing clones impacted by that issue.

Migration of packfiles involves the following steps for each pack:

1. Hardlink (or copy):
   a. the .pack file
   b. the .keep file
   c. the .rev file
2. Move (or copy + delete) the .idx file
3. Delete/unlink:
   a. the .pack file
   b. the .keep file
   c. the .rev file

Moving the index file after the others ensures the pack is not read
from the new cache directory until all associated files (rev, keep)
exist in the cache directory also.

Moving loose objects operates as a move, or copy + delete.

Signed-off-by: Matthew John Cheetham <[email protected]>

Author: mjcheetham

Documentation/git-maintenance.txt

@@ -69,6 +69,7 @@ task:
 * `prefetch`: hourly.
 * `loose-objects`: daily.
 * `incremental-repack`: daily.
+* `cache-local-objects`: weekly.
 --
 +
 `git maintenance register` will also disable foreground maintenance by
@@ -158,6 +159,13 @@ pack-refs::
 	need to iterate across many references. See linkgit:git-pack-refs[1]
 	for more information.
 
+cache-local-objects::
+	The `cache-local-objects` task only operates on Scalar or VFS for Git
+	repositories (cloned with either `scalar clone` or `gvfs clone`) that
+	have the `gvfs.sharedCache` configuration setting present. This task
+	migrates pack files and loose objects from the repository's object
+	directory in to the shared volume cache.
+
 OPTIONS
 -------
 --auto::

builtin/gc.c

@@ -13,6 +13,7 @@
 #define USE_THE_REPOSITORY_VARIABLE
 #define DISABLE_SIGN_COMPARE_WARNINGS
 
+#include "git-compat-util.h"
 #include "builtin.h"
 #include "abspath.h"
 #include "date.h"
@@ -44,6 +45,8 @@
 #include "hook.h"
 #include "setup.h"
 #include "trace2.h"
+#include "copy.h"
+#include "dir.h"
 
 #define FAILED_RUN "failed to run %s"
 
@@ -1360,6 +1363,186 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts
 	return 0;
 }
 
+static void link_or_copy_or_die(const char *src, const char *dst)
+{
+	if (!link(src, dst))
+		return;
+
+	/* Use copy operation if src and dst are on different file systems. */
+	if (errno != EXDEV)
+		warning_errno(_("failed to link '%s' to '%s'"), src, dst);
+
+	if (copy_file(dst, src, 0444))
+		die_errno(_("failed to copy '%s' to '%s'"), src, dst);
+}
+
+static void rename_or_copy_or_die(const char *src, const char *dst)
+{
+	if (!rename(src, dst))
+		return;
+
+	/* Use copy and delete if src and dst are on different file systems. */
+	if (errno != EXDEV)
+		warning_errno(_("failed to move '%s' to '%s'"), src, dst);
+
+	if (copy_file(dst, src, 0444))
+		die_errno(_("failed to copy '%s' to '%s'"), src, dst);
+
+	if (unlink(src))
+		die_errno(_("failed to delete '%s'"), src);
+}
+
+static void migrate_pack(const char *srcdir, const char *dstdir,
+			 const char *pack_filename)
+{
+	size_t basenamelen, srclen, dstlen;
+	struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
+	struct {
+		const char *ext;
+		unsigned move:1;
+	} files[] = {
+		{".pack", 0},
+		{".keep", 0},
+		{".rev", 0},
+		{".idx", 1}, /* The index file must be atomically moved last. */
+	};
+
+	trace2_region_enter("maintenance", "migrate_pack", the_repository);
+
+	basenamelen = strlen(pack_filename) - 5; /* .pack */
+	strbuf_addstr(&src, srcdir);
+	strbuf_addch(&src, '/');
+	strbuf_add(&src, pack_filename, basenamelen);
+	strbuf_addstr(&src, ".idx");
+
+	/* A pack without an index file is not yet ready to be migrated. */
+	if (!file_exists(src.buf))
+		goto cleanup;
+
+	strbuf_setlen(&src, src.len - 4 /* .idx */);
+	strbuf_addstr(&dst, dstdir);
+	strbuf_addch(&dst, '/');
+	strbuf_add(&dst, pack_filename, basenamelen);
+
+	srclen = src.len;
+	dstlen = dst.len;
+
+	/* Move or copy files from the source directory to the destination. */
+	for (size_t i = 0; i < ARRAY_SIZE(files); i++) {
+		strbuf_setlen(&src, srclen);
+		strbuf_addstr(&src, files[i].ext);
+
+		if (!file_exists(src.buf))
+			continue;
+
+		strbuf_setlen(&dst, dstlen);
+		strbuf_addstr(&dst, files[i].ext);
+
+		if (files[i].move)
+			rename_or_copy_or_die(src.buf, dst.buf);
+		else
+			link_or_copy_or_die(src.buf, dst.buf);
+	}
+
+	/*
+	 * Now the pack and all associated files exist at the destination we can
+	 * now clean up the files in the source directory.
+	 */
+	for (size_t i = 0; i < ARRAY_SIZE(files); i++) {
+		/* Files that were moved rather than copied have no clean up. */
+		if (files[i].move)
+			continue;
+
+		strbuf_setlen(&src, srclen);
+		strbuf_addstr(&src, files[i].ext);
+
+		/* Files that never existed in originally have no clean up.*/
+		if (!file_exists(src.buf))
+			continue;
+
+		if (unlink(src.buf))
+			warning_errno(_("failed to delete '%s'"), src.buf);
+	}
+
+cleanup:
+	strbuf_release(&src);
+	strbuf_release(&dst);
+
+	trace2_region_leave("maintenance", "migrate_pack", the_repository);
+}
+
+static void move_pack_to_shared_cache(const char *full_path, size_t full_path_len,
+				      const char *file_name, void *data)
+{
+	char *srcdir;
+	const char *dstdir = (const char *)data;
+
+	/* We only care about the actual pack files here.
+	 * The associated .idx, .keep, .rev files will be copied in tandem
+	 * with the pack file, with the index file being moved last.
+	 * The original locations of the non-index files will only deleted
+	 * once all other files have been copied/moved.
+	 */
+	if (!ends_with(file_name, ".pack"))
+		return;
+
+	srcdir = xstrndup(full_path, full_path_len - strlen(file_name) - 1);
+
+	migrate_pack(srcdir, dstdir, file_name);
+
+	free(srcdir);
+}
+
+static int move_loose_object_to_shared_cache(const struct object_id *oid,
+					     const char *path,
+					     UNUSED void *data)
+{
+	struct stat st;
+	struct strbuf dst = STRBUF_INIT;
+	char *hex = oid_to_hex(oid);
+
+	strbuf_addf(&dst, "%s/%.2s/", shared_object_dir, hex);
+
+	if (stat(dst.buf, &st)) {
+		if (mkdir(dst.buf, 0777))
+			die_errno(_("failed to create directory '%s'"), dst.buf);
+	} else if (!S_ISDIR(st.st_mode))
+		die(_("expected '%s' to be a directory"), dst.buf);
+
+	strbuf_addstr(&dst, hex+2);
+	rename_or_copy_or_die(path, dst.buf);
+
+	strbuf_release(&dst);
+	return 0;
+}
+
+static int maintenance_task_cache_local_objs(UNUSED struct maintenance_run_opts *opts,
+					     UNUSED struct gc_config *cfg)
+{
+	struct strbuf dstdir = STRBUF_INIT;
+	struct repository *r = the_repository;
+
+	/* This task is only applicable with a VFS/Scalar shared cache. */
+	if (!shared_object_dir)
+		return 0;
+
+	/* If the dest is the same as the local odb path then we do nothing. */
+	if (!fspathcmp(r->objects->odb->path, shared_object_dir))
+		goto cleanup;
+
+	strbuf_addf(&dstdir, "%s/pack", shared_object_dir);
+
+	for_each_file_in_pack_dir(r->objects->odb->path, move_pack_to_shared_cache,
+				  dstdir.buf);
+
+	for_each_loose_object(move_loose_object_to_shared_cache, NULL,
+			      FOR_EACH_OBJECT_LOCAL_ONLY);
+
+cleanup:
+	strbuf_release(&dstdir);
+	return 0;
+}
+
 typedef int maintenance_task_fn(struct maintenance_run_opts *opts,
 				struct gc_config *cfg);
 
@@ -1389,6 +1572,7 @@ enum maintenance_task_label {
 	TASK_GC,
 	TASK_COMMIT_GRAPH,
 	TASK_PACK_REFS,
+	TASK_CACHE_LOCAL_OBJS,
 
 	/* Leave as final value */
 	TASK__COUNT
@@ -1425,6 +1609,10 @@ static struct maintenance_task tasks[] = {
 		maintenance_task_pack_refs,
 		pack_refs_condition,
 	},
+	[TASK_CACHE_LOCAL_OBJS] = {
+		"cache-local-objects",
+		maintenance_task_cache_local_objs,
+	},
 };
 
 static int compare_tasks_by_selection(const void *a_, const void *b_)
@@ -1519,6 +1707,8 @@ static void initialize_maintenance_strategy(void)
 		tasks[TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY;
 		tasks[TASK_PACK_REFS].enabled = 1;
 		tasks[TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY;
+		tasks[TASK_CACHE_LOCAL_OBJS].enabled = 1;
+		tasks[TASK_CACHE_LOCAL_OBJS].schedule = SCHEDULE_WEEKLY;
 	}
 }
 

t/t7900-maintenance.sh

@@ -31,6 +31,25 @@ test_systemd_analyze_verify () {
 	fi
 }
 
+test_import_packfile () {
+	printf "blob\ndata <<END\n%s\nEND\n\n" 1 2 3 4 5 | \
+	git -c fastimport.unpackLimit=0 fast-import
+}
+
+test_get_packdir_files() {
+	if [ "$#" -eq 0 ]; then
+		find .git/objects/pack -type f
+	else
+		for arg in "$@"; do
+			find .git/objects/pack -type f -name $arg
+		done
+	fi
+}
+
+test_get_loose_object_files () {
+	find .git/objects -type f -path '.git/objects/??/*'
+}
+
 test_expect_success 'help text' '
 	test_expect_code 129 git maintenance -h >actual &&
 	test_grep "usage: git maintenance <subcommand>" actual &&
@@ -1025,4 +1044,114 @@ test_expect_success 'maintenance aborts with existing lock file' '
 	test_grep "Another scheduled git-maintenance(1) process seems to be running" err
 '
 
+test_expect_success 'cache-local-objects task with no shared cache no op' '
+	test_when_finished "rm -rf repo" &&
+	git init repo &&
+	(
+		cd repo &&
+
+		test_commit something &&
+		git config set maintenance.gc.enabled false &&
+		git config set maintenance.cache-local-objects.enabled true &&
+		git config set maintenance.cache-local-objects.auto 1 &&
+
+		test_import_packfile &&
+		test_get_packdir_files "*.pack" "*.idx" "*.keep" "*.rev" \
+			>files.txt &&
+		test_get_loose_object_files >>files.txt &&
+
+		git maintenance run &&
+		while IFS= read -r f; do
+			test_path_exists $f || exit 1
+		done <files.txt
+	)
+'
+
+test_expect_success 'cache-local-objects task cache path same as local odb no op' '
+	test_when_finished "rm -rf repo" &&
+	git init repo &&
+	(
+		cd repo &&
+
+		test_commit something &&
+		git config set gvfs.sharedcache .git/objects &&
+		git config set maintenance.gc.enabled false &&
+		git config set maintenance.cache-local-objects.enabled true &&
+		git config set maintenance.cache-local-objects.auto 1 &&
+
+		test_import_packfile &&
+		test_get_packdir_files "*.pack" "*.idx" "*.keep" "*.rev" \
+			>files.txt &&
+		test_get_loose_object_files >>files.txt &&
+
+		git maintenance run &&
+		while IFS= read -r f; do
+			test_path_exists $f || exit 1
+		done <files.txt
+	)
+'
+
+test_expect_success 'cache-local-objects task no .rev or .keep' '
+	test_when_finished "rm -rf repo cache" &&
+	mkdir -p cache/pack &&
+	git init repo &&
+	(
+		cd repo &&
+
+		test_commit something &&
+		git config set gvfs.sharedcache ../cache &&
+		git config set maintenance.gc.enabled false &&
+		git config set maintenance.cache-local-objects.enabled true &&
+		git config set maintenance.cache-local-objects.auto 1 &&
+
+		test_import_packfile &&
+		test_get_packdir_files "*.pack" "*.idx" >src.txt &&
+		test_get_loose_object_files >>src.txt &&
+
+		rm -f .git/objects/pack/*.rev .git/objects/pack/*.keep &&
+
+		sed "s/.git\\/objects\\//..\\/cache\\//" src.txt >dst.txt &&
+
+		git maintenance run &&
+		while IFS= read -r f; do
+			test_path_is_missing $f || exit 1
+		done <src.txt &&
+
+		while IFS= read -r f; do
+			test_path_exists $f || exit 1
+		done <dst.txt
+	)
+'
+
+test_expect_success 'cache-local-objects task success' '
+	test_when_finished "rm -rf repo cache" &&
+	mkdir -p cache/pack &&
+	git init repo &&
+	(
+		cd repo &&
+
+		test_commit something &&
+		git config set gvfs.sharedcache ../cache &&
+		git config set maintenance.gc.enabled false &&
+		git config set maintenance.cache-local-objects.enabled true &&
+		git config set maintenance.cache-local-objects.auto 1 &&
+
+		test_import_packfile &&
+		test_get_packdir_files "*.pack" "*.idx" "*.keep" "*.rev" \
+			>src.txt &&
+		test_get_loose_object_files >>src.txt &&
+
+		sed "s/.git\\/objects\\//..\\/cache\\//" src.txt >dst.txt &&
+
+		git maintenance run &&
+		while IFS= read -r f; do
+			test_path_is_missing $f || exit 1
+		done <src.txt &&
+
+		while IFS= read -r f; do
+			test_path_exists $f || exit 1
+		done <dst.txt
+	)
+'
+
 test_done