Merge pull request #315: unpack-trees:virtualfilesystem: Improve efficiency of clear_ce_flags

When the virtualfilesystem is enabled the previous implementation of
clear_ce_flags would iterate all of the cache entries and query whether
each one is in the virtual filesystem to determine whether to clear one
of the SKIP_WORKTREE bits. For each cache entry, we would do a hash
lookup for each parent directory in the is_included_in_virtualfilesystem
function.

The former approach is slow for a typical Windows OS enlistment with
3 million files where only a small percentage is in the virtual
filesystem. The cost is
O(n_index_entries * n_chars_per_path * n_parent_directories_per_path).

In this change, we use the same approach as apply_virtualfilesystem,
which iterates the set of entries in the virtualfilesystem and searches
in the cache for the corresponding entries in order to clear their
flags. This approach has a cost of
O(n_virtual_filesystem_entries * n_chars_per_path * log(n_index_entries)).

The apply_virtualfilesystem code was refactored a bit and modified to
clear flags for all names that 'alias' a given virtual filesystem name
when ignore_case is set.

n_virtual_filesystem_entries is typically much less than
n_index_entries, in which case the new approach is much faster. We wind
up building the name hash for the index, but this occurs quickly thanks
to the multi-threading.

Author: derrickstolee

name-hash.c

@@ -747,6 +747,26 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na
 	return NULL;
 }
 
+struct cache_entry *index_file_next_match(struct index_state *istate, struct cache_entry *ce, int igncase)
+{
+	struct cache_entry *next;
+
+	if (!igncase || !ce) {
+		return NULL;
+	}
+
+	next = hashmap_get_next_entry(&istate->name_hash, ce, ent);
+	if (!next)
+		return NULL;
+
+	hashmap_for_each_entry_from(&istate->name_hash, next, ent) {
+		if (same_name(next, ce->name, ce_namelen(ce), igncase))
+			return next;
+	}
+
+	return NULL;
+}
+
 void free_name_hash(struct index_state *istate)
 {
 	if (!istate->name_hash_initialized)

name-hash.h

@@ -12,6 +12,7 @@ int index_dir_find(struct index_state *istate, const char *name, int namelen,
 
 void adjust_dirname_case(struct index_state *istate, char *name);
 struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase);
+struct cache_entry *index_file_next_match(struct index_state *istate, struct cache_entry *ce, int igncase);
 
 int test_lazy_init_name_hash(struct index_state *istate, int try_threaded);
 void add_name_hash(struct index_state *istate, struct cache_entry *ce);

unpack-trees.c

@@ -1722,14 +1722,6 @@ static int clear_ce_flags_1(struct index_state *istate,
 			continue;
 		}
 
-		/* if it's not in the virtual file system, exit early */
-		if (core_virtualfilesystem) {
-			if (is_included_in_virtualfilesystem(ce->name, ce->ce_namelen) > 0)
-				ce->ce_flags &= ~clear_mask;
-			cache++;
-			continue;
-		}
-
 		if (prefix->len && strncmp(ce->name, prefix->buf, prefix->len))
 			break;
 
@@ -1806,12 +1798,19 @@ static int clear_ce_flags(struct index_state *istate,
 	xsnprintf(label, sizeof(label), "clear_ce_flags/0x%08lx_0x%08lx",
 		  (unsigned long)select_mask, (unsigned long)clear_mask);
 	trace2_region_enter("unpack_trees", label, the_repository);
-	rval = clear_ce_flags_1(istate,
-				istate->cache,
-				istate->cache_nr,
-				&prefix,
-				select_mask, clear_mask,
-				pl, 0, 0);
+	if (core_virtualfilesystem) {
+		rval = clear_ce_flags_virtualfilesystem(istate,
+							select_mask,
+							clear_mask);
+	} else {
+		rval = clear_ce_flags_1(istate,
+					istate->cache,
+					istate->cache_nr,
+					&prefix,
+					select_mask, clear_mask,
+					pl, 0, 0);
+	}
+
 	trace2_region_leave("unpack_trees", label, the_repository);
 
 	stop_progress(&istate->progress);

virtualfilesystem.c

@@ -252,93 +252,133 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt
 	return -1;
 }
 
-/*
- * Update the CE_SKIP_WORKTREE bits based on the virtual file system.
- */
-void apply_virtualfilesystem(struct index_state *istate)
+struct apply_virtual_filesystem_stats {
+	int nr_unknown;
+	int nr_vfs_dirs;
+	int nr_vfs_rows;
+	int nr_bulk_skip;
+	int nr_explicit_skip;
+};
+
+static void clear_ce_flags_virtualfilesystem_1(struct index_state *istate, int select_mask, int clear_mask,
+					       struct apply_virtual_filesystem_stats *stats)
 {
 	char *buf, *entry;
 	size_t i;
-	int nr_unknown = 0;
-	int nr_vfs_dirs = 0;
-	int nr_vfs_rows = 0;
-	int nr_bulk_skip = 0;
-	int nr_explicit_skip = 0;
-
-	if (!repo_config_get_virtualfilesystem(istate->repo))
-		return;
-
-	trace2_region_enter("vfs", "apply", the_repository);
 
 	if (!virtual_filesystem_data.len)
 		get_virtual_filesystem_data(istate->repo, &virtual_filesystem_data);
 
-	/* set CE_SKIP_WORKTREE bit on all entries */
-	for (i = 0; i < istate->cache_nr; i++)
-		istate->cache[i]->ce_flags |= CE_SKIP_WORKTREE;
-
-	/* clear CE_SKIP_WORKTREE bit for everything in the virtual file system */
+	/* clear specified flag bits for everything in the virtual file system */
 	entry = buf = virtual_filesystem_data.buf;
 	for (i = 0; i < virtual_filesystem_data.len; i++) {
 		if (buf[i] == '\0') {
+			struct cache_entry *ce;
 			ssize_t pos, len;
 
-			nr_vfs_rows++;
+			stats->nr_vfs_rows++;
 
 			len = buf + i - entry;
 
 			/* look for a directory wild card (ie "dir1/") */
 			if (buf[i - 1] == '/') {
-				nr_vfs_dirs++;
+				stats->nr_vfs_dirs++;
 				if (ignore_case)
 					adjust_dirname_case(istate, entry);
 				pos = index_name_pos(istate, entry, len);
 				if (pos < 0) {
-					pos = -pos - 1;
-					while ((size_t)pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) {
-						if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE)
-							nr_bulk_skip++;
-						istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE;
-						pos++;
+					for (pos = -pos - 1; (size_t)pos < istate->cache_nr; pos++) {
+						ce = istate->cache[pos];
+						if (fspathncmp(ce->name, entry, len))
+							break;
+
+						if (select_mask && !(ce->ce_flags & select_mask))
+							continue;
+
+						if (ce->ce_flags & clear_mask)
+							stats->nr_bulk_skip++;
+						ce->ce_flags &= ~clear_mask;
 					}
 				}
 			} else {
 				if (ignore_case) {
-					struct cache_entry *ce = index_file_exists(istate, entry, len, ignore_case);
-					if (ce) {
-						if (ce->ce_flags & CE_SKIP_WORKTREE)
-							nr_explicit_skip++;
-						ce->ce_flags &= ~CE_SKIP_WORKTREE;
-					}
-					else {
-						nr_unknown++;
-					}
+					ce = index_file_exists(istate, entry, len, ignore_case);
 				} else {
 					int pos = index_name_pos(istate, entry, len);
-					if (pos >= 0) {
-						if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE)
-							nr_explicit_skip++;
-						istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE;
-					}
-					else {
-						nr_unknown++;
-					}
+
+					ce = NULL;
+					if (pos >= 0)
+						ce = istate->cache[pos];
+				}
+
+				if (ce) {
+					do {
+						if (!select_mask || (ce->ce_flags & select_mask)) {
+							if (ce->ce_flags & clear_mask)
+								stats->nr_explicit_skip++;
+							ce->ce_flags &= ~clear_mask;
+						}
+
+						/*
+						 * There may be aliases with different cases of the same
+						 * name that also need to be modified.
+						 */
+						if (ignore_case)
+							ce = index_file_next_match(istate, ce, ignore_case);
+						else
+							break;
+
+					} while (ce);
+				} else {
+					stats->nr_unknown++;
 				}
 			}
 
 			entry += len + 1;
 		}
 	}
+}
+
+/*
+ * Clear the specified flags for all entries in the virtual file system
+ * that match the specified select mask. Returns the number of entries
+ * processed.
+ */
+int clear_ce_flags_virtualfilesystem(struct index_state *istate, int select_mask, int clear_mask)
+{
+	struct apply_virtual_filesystem_stats stats = {0};
+
+	clear_ce_flags_virtualfilesystem_1(istate, select_mask, clear_mask, &stats);
+	return istate->cache_nr;
+}
+
+/*
+ * Update the CE_SKIP_WORKTREE bits based on the virtual file system.
+ */
+void apply_virtualfilesystem(struct index_state *istate)
+{
+	size_t i;
+	struct apply_virtual_filesystem_stats stats = {0};
+
+	if (!repo_config_get_virtualfilesystem(istate->repo))
+		return;
+
+	trace2_region_enter("vfs", "apply", the_repository);
+
+	/* set CE_SKIP_WORKTREE bit on all entries */
+	for (i = 0; i < istate->cache_nr; i++)
+		istate->cache[i]->ce_flags |= CE_SKIP_WORKTREE;
 
-	if (nr_vfs_rows > 0) {
-		trace2_data_intmax("vfs", the_repository, "apply/tracked", nr_bulk_skip + nr_explicit_skip);
+	clear_ce_flags_virtualfilesystem_1(istate, 0, CE_SKIP_WORKTREE, &stats);
+	if (stats.nr_vfs_rows > 0) {
+		trace2_data_intmax("vfs", the_repository, "apply/tracked", stats.nr_bulk_skip + stats.nr_explicit_skip);
 
-		trace2_data_intmax("vfs", the_repository, "apply/vfs_rows", nr_vfs_rows);
-		trace2_data_intmax("vfs", the_repository, "apply/vfs_dirs", nr_vfs_dirs);
+		trace2_data_intmax("vfs", the_repository, "apply/vfs_rows", stats.nr_vfs_rows);
+		trace2_data_intmax("vfs", the_repository, "apply/vfs_dirs", stats.nr_vfs_dirs);
 
-		trace2_data_intmax("vfs", the_repository, "apply/nr_unknown", nr_unknown);
-		trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", nr_bulk_skip);
-		trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", nr_explicit_skip);
+		trace2_data_intmax("vfs", the_repository, "apply/nr_unknown", stats.nr_unknown);
+		trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", stats.nr_bulk_skip);
+		trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", stats.nr_explicit_skip);
 	}
 
 	trace2_region_leave("vfs", "apply", the_repository);

virtualfilesystem.h

@@ -6,6 +6,13 @@
  */
 void apply_virtualfilesystem(struct index_state *istate);
 
+/*
+ * Clear the specified flags for all entries in the virtual file system
+ * that match the specified select mask. Returns the number of entries
+ * processed.
+ */
+int clear_ce_flags_virtualfilesystem(struct index_state *istate, int select_mask, int clear_mask);
+
 /*
  * Return 1 if the requested item is found in the virtual file system,
  * 0 for not found and -1 for undecided.