maintenance: add new vfs-cache-move maintenance task

Introduce a new maintenance task, `vfs-cache-move`, that operates on
Scalar or VFS for Git repositories with a per-volume, shared object
cache (specified by `gvfs.sharedCache`) to migrate packfiles from the
repository object directory to the shared cache.

Older versions of `microsoft/git` incorrectly placed packfiles in the
repository object directory instead of the shared cache; this task will
help clean up existing clones impacted by that issue.

Signed-off-by: Matthew John Cheetham <[email protected]>

Author: mjcheetham

Documentation/git-maintenance.txt

@@ -69,6 +69,7 @@ task:
 * `prefetch`: hourly.
 * `loose-objects`: daily.
 * `incremental-repack`: daily.
+* `vfs-cache-move`: weekly.
 --
 +
 `git maintenance register` will also disable foreground maintenance by
@@ -158,6 +159,13 @@ pack-refs::
 	need to iterate across many references. See linkgit:git-pack-refs[1]
 	for more information.
 
+vfs-cache-move::
+	The `vfs-cache-move` task operates only Scalar or VFS for Git
+	repositories (cloned with either `scalar clone` or `gvfs clone`) that
+	have the `gvfs.sharedCache` configuration setting present. This task
+	migrates pack files from the repository's object directory in to the
+	shared volume cache.
+
 OPTIONS
 -------
 --auto::

builtin/gc.c

@@ -1345,6 +1345,155 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts
 	return 0;
 }
 
+
+static void copy_file(const char *srcdir, const char *dstdir, const char *name)
+{
+	int ret = 0;
+	struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
+	int srcfd = -1, dstfd = -1;
+	char buf[1024];
+	ssize_t nr;
+
+	strbuf_addf(&src, "%s/%s", srcdir, name);
+	strbuf_addf(&dst, "%s/%s", dstdir, name);
+
+	srcfd = open(src.buf, O_RDONLY);
+	if (srcfd < 0) {
+		error("failed to open source file");
+		ret = 1;
+		goto cleanup;
+	}
+
+	dstfd = open(dst.buf, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+	if (dstfd < 0) {
+		error("failed to open destination file");
+		ret = 1;
+		goto cleanup;
+	}
+
+	while ((nr = read(srcfd, buf, sizeof(buf))) > 0) {
+		if (write(dstfd, buf, nr) < 0) {
+			error("failed to write to destination file");
+			ret = 1;
+		}
+	}
+
+cleanup:
+	if (srcfd >= 0) close(srcfd);
+	if (dstfd >= 0) close(dstfd);
+
+	if (ret)
+		die_errno(_("failed to copy '%s' to '%s'"), src.buf, dst.buf);
+
+	strbuf_release(&src);
+	strbuf_release(&dst);
+}
+
+static void move_file(const char *srcdir, const char *dstdir, const char *name)
+{
+	struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
+
+	strbuf_addf(&src, "%s/%s", srcdir, name);
+	strbuf_addf(&dst, "%s/%s", dstdir, name);
+
+	if (rename(src.buf, dst.buf))
+		die_errno(_("failed to move '%s' to '%s'"), src.buf, dst.buf);
+
+	strbuf_release(&src);
+	strbuf_release(&dst);
+}
+
+static void delete_file(const char *dir, const char *name)
+{
+	struct strbuf path = STRBUF_INIT;
+
+	strbuf_addf(&path, "%s/%s", dir, name);
+
+	if (unlink(path.buf))
+		warning_errno(_("failed to delete '%s'"), path.buf);
+
+	strbuf_release(&path);
+}
+
+static void migrate_pack(const char *srcdir, const char *dstdir,
+			 const char *pack_filename)
+{
+	struct strbuf path = STRBUF_INIT;
+	struct stat st;
+	char *basename, *keep_filename, *rev_filename, *idx_filename;
+	int has_keep, has_rev;
+
+	basename = xstrndup(pack_filename, strlen(pack_filename) - 5 /*.pack*/);
+	keep_filename = xstrfmt("%s.keep", basename);
+	rev_filename = xstrfmt("%s.rev", basename);
+	idx_filename = xstrfmt("%s.idx", basename);
+
+	strbuf_addf(&path, "%s/%s", srcdir, keep_filename);
+	has_keep = !stat(path.buf, &st);
+	strbuf_reset(&path);
+	strbuf_addf(&path, "%s/%s", srcdir, rev_filename);
+	has_rev = !stat(path.buf, &st);
+	strbuf_release(&path);
+
+	/* Copy all but the index file, which we will *move* atomically */
+	copy_file(srcdir, dstdir, pack_filename);
+	if (has_keep) copy_file(srcdir, dstdir, keep_filename);
+	if (has_rev) copy_file(srcdir, dstdir, rev_filename);
+	move_file(srcdir, dstdir, idx_filename);
+
+	/*
+	 * Now the pack and associated files exist at the destination we
+	 * we can now clean up files in the source directory.
+	 */
+	delete_file(srcdir, pack_filename);
+	if (has_keep) delete_file(srcdir, keep_filename);
+	if (has_rev) delete_file(srcdir, rev_filename);
+
+	free(idx_filename);
+	free(keep_filename);
+	free(rev_filename);
+}
+
+static void move_pack_to_vfs_cache(const char *full_path, size_t full_path_len,
+				   const char *file_name, UNUSED void *data)
+{
+	char *srcdir;
+	struct strbuf dstdir = STRBUF_INIT;
+
+	/* We only care about the actual pack files here.
+	 * The associated .idx, .keep, .rev files will be copied in tandem
+	 * with the pack file, with the index file being moved last.
+	 * The original locations of the non-index files will only deleted
+	 * once all other files have been copied/moved.
+	 */
+	if (!ends_with(file_name, ".pack"))
+		return;
+
+	srcdir = xstrndup(full_path, full_path_len - strlen(file_name) - 1);
+
+	/* No cache or same source + desintation means there's no work to do. */
+	if (!object_dir || !fspathcmp(srcdir, object_dir))
+		return;
+
+	strbuf_addf(&dstdir, "%s/pack", object_dir);
+
+	migrate_pack(srcdir, dstdir.buf, file_name);
+
+	free(srcdir);
+	strbuf_release(&dstdir);
+}
+
+static int maintenance_task_vfs_cache_move(UNUSED struct maintenance_run_opts *opts,
+					   UNUSED struct gc_config *cfg)
+{
+	struct repository *r = the_repository;
+
+	for_each_file_in_pack_dir(r->objects->odb->path, move_pack_to_vfs_cache,
+				  NULL);
+
+	return 0;
+}
+
 typedef int maintenance_task_fn(struct maintenance_run_opts *opts,
 				struct gc_config *cfg);
 
@@ -1374,6 +1523,7 @@ enum maintenance_task_label {
 	TASK_GC,
 	TASK_COMMIT_GRAPH,
 	TASK_PACK_REFS,
+	TASK_VFS_CACHE_MOVE,
 
 	/* Leave as final value */
 	TASK__COUNT
@@ -1410,6 +1560,10 @@ static struct maintenance_task tasks[] = {
 		maintenance_task_pack_refs,
 		pack_refs_condition,
 	},
+	[TASK_VFS_CACHE_MOVE] = {
+		"vfs-cache-move",
+		maintenance_task_vfs_cache_move,
+	},
 };
 
 static int compare_tasks_by_selection(const void *a_, const void *b_)
@@ -1504,6 +1658,8 @@ static void initialize_maintenance_strategy(void)
 		tasks[TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY;
 		tasks[TASK_PACK_REFS].enabled = 1;
 		tasks[TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY;
+		tasks[TASK_VFS_CACHE_MOVE].enabled = 1;
+		tasks[TASK_VFS_CACHE_MOVE].schedule = SCHEDULE_WEEKLY;
 	}
 }
 

t/t7900-maintenance.sh

@@ -1011,4 +1011,34 @@ test_expect_success 'repacking loose objects is quiet' '
 	)
 '
 
+test_expect_success 'vfs-cache-move task' '
+	#test_when_finished "rm -rf repo" &&
+	git init repo &&
+	(
+		cd repo &&
+
+		test_commit something &&
+		git config set gvfs.sharedcache ../cache &&
+		git config set maintenance.gc.enabled false &&
+		git config set maintenance.vfs-cache-move.enabled true &&
+		git config set maintenance.vfs-cache-move.auto 1 &&
+
+		touch .git/objects/pack/vfs-12345678.pack &&
+		touch .git/objects/pack/vfs-12345678.keep &&
+		touch .git/objects/pack/vfs-12345678.rev &&
+		touch .git/objects/pack/vfs-12345678.idx &&
+		mkdir -p ../cache/pack &&
+
+		git maintenance run &&
+		test_path_is_missing .git/objects/pack/vfs-12345678.pack &&
+		test_path_is_missing .git/objects/pack/vfs-12345678.keep &&
+		test_path_is_missing .git/objects/pack/vfs-12345678.rev &&
+		test_path_is_missing .git/objects/pack/vfs-12345678.idx &&
+		test_path_exists ../cache/pack/vfs-12345678.pack &&
+		test_path_exists ../cache/pack/vfs-12345678.keep &&
+		test_path_exists ../cache/pack/vfs-12345678.rev &&
+		test_path_exists ../cache/pack/vfs-12345678.idx
+	)
+'
+
 test_done