Merge pull request #23 from benpeart/unpack-trees-with-cache-tree-gvfs

Unpack trees with cache tree gvfs

Author: benpeart

cache-tree.c

@@ -4,6 +4,7 @@
 #include "tree-walk.h"
 #include "cache-tree.h"
 #include "object-store.h"
+#include "replace-object.h"
 #include "gvfs.h"
 
 #ifndef DEBUG_CACHE_TREE
@@ -457,7 +458,9 @@ int cache_tree_update(struct index_state *istate, int flags)
 
 	if (i)
 		return i;
+	trace_performance_enter();
 	i = update_one(it, cache, entries, "", 0, &skip, flags);
+	trace_performance_leave("cache_tree_update");
 	if (i < 0)
 		return i;
 	istate->cache_changed |= CACHE_TREE_CHANGED;
@@ -742,3 +745,80 @@ int cache_tree_matches_traversal(struct cache_tree *root,
 		return it->entry_count;
 	return 0;
 }
+
+static void verify_one(struct index_state *istate,
+		       struct cache_tree *it,
+		       struct strbuf *path)
+{
+	int i, pos, len = path->len;
+	struct strbuf tree_buf = STRBUF_INIT;
+	struct object_id new_oid;
+
+	for (i = 0; i < it->subtree_nr; i++) {
+		strbuf_addf(path, "%s/", it->down[i]->name);
+		verify_one(istate, it->down[i]->cache_tree, path);
+		strbuf_setlen(path, len);
+	}
+
+	if (it->entry_count < 0 ||
+	    /* no verification on tests (t7003) that replace trees */
+	    lookup_replace_object(the_repository, &it->oid) != &it->oid)
+		return;
+
+	if (path->len) {
+		pos = index_name_pos(istate, path->buf, path->len);
+		pos = -pos - 1;
+	} else {
+		pos = 0;
+	}
+
+	i = 0;
+	while (i < it->entry_count) {
+		struct cache_entry *ce = istate->cache[pos + i];
+		const char *slash;
+		struct cache_tree_sub *sub = NULL;
+		const struct object_id *oid;
+		const char *name;
+		unsigned mode;
+		int entlen;
+
+		if (ce->ce_flags & (CE_STAGEMASK | CE_INTENT_TO_ADD | CE_REMOVE))
+			BUG("%s with flags 0x%x should not be in cache-tree",
+			    ce->name, ce->ce_flags);
+		name = ce->name + path->len;
+		slash = strchr(name, '/');
+		if (slash) {
+			entlen = slash - name;
+			sub = find_subtree(it, ce->name + path->len, entlen, 0);
+			if (!sub || sub->cache_tree->entry_count < 0)
+				BUG("bad subtree '%.*s'", entlen, name);
+			oid = &sub->cache_tree->oid;
+			mode = S_IFDIR;
+			i += sub->cache_tree->entry_count;
+		} else {
+			oid = &ce->oid;
+			mode = ce->ce_mode;
+			entlen = ce_namelen(ce) - path->len;
+			i++;
+		}
+		strbuf_addf(&tree_buf, "%o %.*s%c", mode, entlen, name, '\0');
+		strbuf_add(&tree_buf, oid->hash, the_hash_algo->rawsz);
+	}
+	hash_object_file(tree_buf.buf, tree_buf.len, tree_type, &new_oid);
+	if (oidcmp(&new_oid, &it->oid))
+		BUG("cache-tree for path %.*s does not match. "
+		    "Expected %s got %s", len, path->buf,
+		    oid_to_hex(&new_oid), oid_to_hex(&it->oid));
+	strbuf_setlen(path, len);
+	strbuf_release(&tree_buf);
+}
+
+void cache_tree_verify(struct index_state *istate)
+{
+	struct strbuf path = STRBUF_INIT;
+
+	if (!istate->cache_tree)
+		return;
+	verify_one(istate, istate->cache_tree, &path);
+	strbuf_release(&path);
+}

cache-tree.h

@@ -32,6 +32,7 @@ struct cache_tree *cache_tree_read(const char *buffer, unsigned long size);
 
 int cache_tree_fully_valid(struct cache_tree *);
 int cache_tree_update(struct index_state *, int);
+void cache_tree_verify(struct index_state *);
 
 /* bitmasks to write_cache_as_tree flags */
 #define WRITE_TREE_MISSING_OK 1

diff-lib.c

@@ -518,7 +518,6 @@ static int diff_cache(struct rev_info *revs,
 int run_diff_index(struct rev_info *revs, int cached)
 {
 	struct object_array_entry *ent;
-	uint64_t start = getnanotime();
 
 	if (revs->pending.nr != 1)
 		BUG("run_diff_index must be passed exactly one tree");
@@ -531,7 +530,7 @@ int run_diff_index(struct rev_info *revs, int cached)
 	diffcore_fix_diff_index(&revs->diffopt);
 	diffcore_std(&revs->diffopt);
 	diff_flush(&revs->diffopt);
-	trace_performance_since(start, "diff-index");
+	trace_performance_leave("diff-index");
 	return 0;
 }
 

dir.c

@@ -2341,10 +2341,13 @@ int read_directory(struct dir_struct *dir, struct index_state *istate,
 		   const char *path, int len, const struct pathspec *pathspec)
 {
 	struct untracked_cache_dir *untracked;
-	uint64_t start = getnanotime();
 
-	if (has_symlink_leading_path(path, len))
+	trace_performance_enter();
+
+	if (has_symlink_leading_path(path, len)) {
+		trace_performance_leave("read directory %.*s", len, path);
 		return dir->nr;
+	}
 
 	untracked = validate_untracked_cache(dir, len, pathspec);
 	if (!untracked)
@@ -2380,7 +2383,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate,
 		dir->nr = i;
 	}
 
-	trace_performance_since(start, "read directory %.*s", len, path);
+	trace_performance_leave("read directory %.*s", len, path);
 	if (dir->untracked) {
 		static int force_untracked_cache = -1;
 		static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS);

name-hash.c

@@ -578,10 +578,10 @@ static void threaded_lazy_init_name_hash(
 
 static void lazy_init_name_hash(struct index_state *istate)
 {
-	uint64_t start = getnanotime();
 
 	if (istate->name_hash_initialized)
 		return;
+	trace_performance_enter();
 	hashmap_init(&istate->name_hash, cache_entry_cmp, NULL, istate->cache_nr);
 	hashmap_init(&istate->dir_hash, dir_entry_cmp, NULL, istate->cache_nr);
 
@@ -602,7 +602,7 @@ static void lazy_init_name_hash(struct index_state *istate)
 	}
 
 	istate->name_hash_initialized = 1;
-	trace_performance_since(start, "initialize name hash");
+	trace_performance_leave("initialize name hash");
 }
 
 /*

preload-index.c

@@ -76,7 +76,6 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec)
 {
 	int threads, i, work, offset;
 	struct thread_data data[MAX_PARALLEL];
-	uint64_t start = getnanotime();
 
 	if (!core_preload_index)
 		return;
@@ -86,6 +85,7 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec)
 		threads = 2;
 	if (threads < 2)
 		return;
+	trace_performance_enter();
 	if (threads > MAX_PARALLEL)
 		threads = MAX_PARALLEL;
 	offset = 0;
@@ -108,7 +108,7 @@ void preload_index(struct index_state *index, const struct pathspec *pathspec)
 		if (pthread_join(p->pthread, NULL))
 			die("unable to join threaded lstat");
 	}
-	trace_performance_since(start, "preload index");
+	trace_performance_leave("preload index");
 	enable_fscache(0);
 }
 #endif

read-cache.c

@@ -1478,8 +1478,8 @@ int refresh_index(struct index_state *istate, unsigned int flags,
 	const char *typechange_fmt;
 	const char *added_fmt;
 	const char *unmerged_fmt;
-	uint64_t start = getnanotime();
 
+	trace_performance_enter();
 	modified_fmt = (in_porcelain ? "M\t%s\n" : "%s: needs update\n");
 	deleted_fmt = (in_porcelain ? "D\t%s\n" : "%s: needs update\n");
 	typechange_fmt = (in_porcelain ? "T\t%s\n" : "%s needs update\n");
@@ -1551,7 +1551,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
 		replace_index_entry(istate, i, new_entry);
 	}
 	enable_fscache(0);
-	trace_performance_since(start, "refresh index");
+	trace_performance_leave("refresh index");
 	return has_errors;
 }
 
@@ -2007,7 +2007,6 @@ static void freshen_shared_index(const char *shared_index, int warn)
 int read_index_from(struct index_state *istate, const char *path,
 		    const char *gitdir)
 {
-	uint64_t start = getnanotime();
 	struct split_index *split_index;
 	int ret;
 	char *base_oid_hex;
@@ -2017,15 +2016,17 @@ int read_index_from(struct index_state *istate, const char *path,
 	if (istate->initialized)
 		return istate->cache_nr;
 
+	trace_performance_enter();
 	ret = do_read_index(istate, path, 0);
-	trace_performance_since(start, "read cache %s", path);
+	trace_performance_leave("read cache %s", path);
 
 	split_index = istate->split_index;
 	if (!split_index || is_null_oid(&split_index->base_oid)) {
 		post_read_index_from(istate);
 		return ret;
 	}
 
+	trace_performance_enter();
 	if (split_index->base)
 		discard_index(split_index->base);
 	else
@@ -2042,8 +2043,8 @@ int read_index_from(struct index_state *istate, const char *path,
 	freshen_shared_index(base_path, 0);
 	merge_base_index(istate);
 	post_read_index_from(istate);
-	trace_performance_since(start, "read cache %s", base_path);
 	free(base_path);
+	trace_performance_leave("read cache %s", base_path);
 	return ret;
 }
 
@@ -2752,6 +2753,9 @@ int write_locked_index(struct index_state *istate, struct lock_file *lock,
 	int new_shared_index, ret;
 	struct split_index *si = istate->split_index;
 
+	if (git_env_bool("GIT_TEST_CHECK_CACHE_TREE", 0))
+		cache_tree_verify(istate);
+
 	if ((flags & SKIP_IF_UNCHANGED) && !istate->cache_changed) {
 		if (flags & COMMIT_LOCK)
 			rollback_lock_file(lock);
@@ -2948,6 +2952,8 @@ void move_index_extensions(struct index_state *dst, struct index_state *src)
 {
 	dst->untracked = src->untracked;
 	src->untracked = NULL;
+	dst->cache_tree = src->cache_tree;
+	src->cache_tree = NULL;
 }
 
 struct cache_entry *dup_cache_entry(const struct cache_entry *ce,

t/README

@@ -319,6 +319,10 @@ GIT_TEST_OE_DELTA_SIZE=<n> exercises the uncomon pack-objects code
 path where deltas larger than this limit require extra memory
 allocation for bookkeeping.
 
+GIT_TEST_VALIDATE_INDEX_CACHE_ENTRIES=<boolean> checks that cache-tree
+records are valid when the index is written out or after a merge. This
+is mostly to catch missing invalidation. Default is true.
+
 Naming Tests
 ------------
 

t/test-lib.sh

@@ -1256,6 +1256,12 @@ else
 	test_set_prereq C_LOCALE_OUTPUT
 fi
 
+if test -z "$GIT_TEST_CHECK_CACHE_TREE"
+then
+	GIT_TEST_CHECK_CACHE_TREE=true
+	export GIT_TEST_CHECK_CACHE_TREE
+fi
+
 test_lazy_prereq PIPE '
 	# test whether the filesystem supports FIFOs
 	test_have_prereq !MINGW,!CYGWIN &&