connected: add incremental connectivity check

Documentation/config/transfer.adoc

@@ -1,3 +1,25 @@
+transfer.connectivityCheck::
+	Choose which algorithm to use for the connectivity check
+	performed during object transfer operations such as
+	linkgit:git-fetch[1] and linkgit:git-receive-pack[1].
+	The connectivity check verifies that all objects reachable
+	from the incoming tips are available locally or, in a partial
+	clone, promised by a promisor remote.
+	The variants are as follows:
++
+--
+`rev-list` (default);;
+	Delegate to `rev-list --objects --not --all`.  This walks
+	the full object closure of the boundary commits.
+`incremental`;;
+	Verify incoming commits by diffing their trees against parent
+	trees, recursively descending only into entries that differ.
+	The largest benefits occur when incoming commits change a
+	small fraction of a large tree closure.
+	Falls back to `rev-list` when replacement objects are
+	active or a deepening fetch is in progress.
+--
+
 transfer.credentialsInUrl::
 	A configured URL can contain plaintext credentials in the form
 	`<protocol>://<user>:<password>@<domain>/<path>`. You may want

Makefile

@@ -812,6 +812,7 @@ TEST_BUILTINS_OBJS += test-bitmap.o
 TEST_BUILTINS_OBJS += test-bloom.o
 TEST_BUILTINS_OBJS += test-bundle-uri.o
 TEST_BUILTINS_OBJS += test-cache-tree.o
+TEST_BUILTINS_OBJS += test-check-connected.o
 TEST_BUILTINS_OBJS += test-chmtime.o
 TEST_BUILTINS_OBJS += test-config.o
 TEST_BUILTINS_OBJS += test-crontab.o

builtin/fetch.c

@@ -1247,14 +1247,23 @@ static int store_updated_refs(struct display_state *display_state,
 
 	if (!connectivity_checked) {
 		struct check_connected_options opt = CHECK_CONNECTED_INIT;
+		struct ref *r;
+
+		for (r = ref_map; r; r = r->next) {
+			if (r->peer_ref && !is_null_oid(&r->peer_ref->old_oid))
+				oid_array_append(&opt.old_tips,
+						 &r->peer_ref->old_oid);
+		}
 
 		opt.exclude_hidden_refs_section = "fetch";
 		rm = ref_map;
 		if (check_connected(iterate_ref_map, &rm, &opt)) {
 			rc = error(_("%s did not send all necessary objects"),
 				   display_state->url);
+			oid_array_clear(&opt.old_tips);
 			goto abort;
 		}
+		oid_array_clear(&opt.old_tips);
 	}
 
 	/*
@@ -1391,6 +1400,7 @@ static int check_exist_and_connected(struct ref *ref_map)
 	struct ref *rm = ref_map;
 	struct check_connected_options opt = CHECK_CONNECTED_INIT;
 	struct ref *r;
+	int ret;
 
 	/*
 	 * If we are deepening a shallow clone we already have these
@@ -1420,9 +1430,17 @@ static int check_exist_and_connected(struct ref *ref_map)
 			return -1;
 	}
 
+	for (r = rm; r; r = r->next) {
+		if (r->peer_ref && !is_null_oid(&r->peer_ref->old_oid))
+			oid_array_append(&opt.old_tips,
+					 &r->peer_ref->old_oid);
+	}
+
 	opt.quiet = 1;
 	opt.exclude_hidden_refs_section = "fetch";
-	return check_connected(iterate_ref_map, &rm, &opt);
+	ret = check_connected(iterate_ref_map, &rm, &opt);
+	oid_array_clear(&opt.old_tips);
+	return ret;
 }
 
 static int fetch_and_consume_refs(struct display_state *display_state,

builtin/receive-pack.c

@@ -2059,6 +2059,14 @@ static void execute_commands(struct command *commands,
 			/* ...else, continue without relaying sideband */
 		}
 
+		for (cmd = commands; cmd; cmd = cmd->next) {
+			if (!is_null_oid(&cmd->old_oid) &&
+			    !is_null_oid(&cmd->new_oid) &&
+			    !cmd->skip_update)
+				oid_array_append(&opt.old_tips,
+						 &cmd->old_oid);
+		}
+
 		data.cmds = commands;
 		data.si = si;
 		opt.err_fd = err_fd;
@@ -2074,6 +2082,7 @@ static void execute_commands(struct command *commands,
 			finish_async(&muxer);
 
 		strvec_clear(&env);
+		oid_array_clear(&opt.old_tips);
 	}
 
 	reject_updates_to_hidden(commands);

commit-reach.c

@@ -276,6 +276,113 @@ static int paint_down_to_common(struct repository *r,
 	return 0;
 }
 
+static void commit_list_drop_flags(struct commit_list **listp, unsigned flags)
+{
+	while (*listp) {
+		if ((*listp)->item->object.flags & flags) {
+			struct commit_list *entry = *listp;
+			*listp = entry->next;
+			free(entry);
+		} else {
+			listp = &(*listp)->next;
+		}
+	}
+}
+
+/*
+ * Paint commits reachable from 'tips' with PARENT1 and commits
+ * reachable from 'bases' with PARENT2, then collect the commits
+ * painted only with PARENT1 (reachable exclusively from tips)
+ * into new_commits.
+ *
+ * Uses the commit-graph's corrected commit dates for generation
+ * ordering.  Bails out once the walk drops below the generation
+ * where incoming commits enter the commit-graph, as the narrow
+ * seed set may not converge efficiently below that point.
+ *
+ * Returns -1 to signal the caller to fall back to another method
+ * when no usable commit-graph is available or when the walk
+ * cannot find a boundary efficiently.
+ */
+int repo_find_boundary_commits(struct repository *r,
+			       struct commit **bases, size_t nr_bases,
+			       size_t nr_tips, struct commit **tips,
+			       struct commit_list **new_commits)
+{
+	struct paint_state state = {
+		.queue = { compare_commits_by_gen_then_commit_date }
+	};
+	struct commit *commit;
+	timestamp_t gen_floor = GENERATION_NUMBER_INFINITY;
+	int ret = -1;
+	size_t i;
+
+	if (!corrected_commit_dates_enabled(r))
+		return -1;
+
+	state.last_gen = GENERATION_NUMBER_INFINITY;
+	state.topo_ceiling = GENERATION_NUMBER_INFINITY;
+
+	for (i = 0; i < nr_tips; i++)
+		paint_queue_put(&state, tips[i], PARENT1);
+	for (i = 0; i < nr_bases; i++)
+		paint_queue_put(&state, bases[i], PARENT2);
+
+	while ((commit = paint_queue_get(&state))) {
+		struct commit_list *parents;
+		unsigned flags;
+		timestamp_t gen = commit_graph_generation(commit);
+
+		flags = commit->object.flags & (PARENT1 | PARENT2);
+
+		if (flags == PARENT1)
+			commit_list_insert(commit, new_commits);
+
+		for (parents = commit->parents; parents; parents = parents->next) {
+			struct commit *p = parents->item;
+			if ((p->object.flags & flags) == flags)
+				continue;
+			if (repo_parse_commit(r, p))
+				goto done;
+			if (flags == PARENT1 &&
+			    gen == GENERATION_NUMBER_INFINITY) {
+				timestamp_t pgen = commit_graph_generation(p);
+				if (pgen < gen_floor)
+					gen_floor = pgen;
+			}
+			paint_queue_put(&state, p, flags);
+		}
+
+		/*
+		 * gen_floor is the lowest generation where an incoming
+		 * commit (PARENT1, gen=INFINITY) first enters the
+		 * commit-graph.  Below this point the narrow seed set
+		 * may not converge efficiently.
+		 */
+		if (!state.parent1_count || gen < gen_floor)
+			break;
+	}
+
+	if (state.parent1_count)
+		goto done;
+
+	commit_list_drop_flags(new_commits, PARENT2);
+
+	ret = 0;
+
+done:
+	clear_prio_queue(&state.queue);
+	for (i = 0; i < nr_tips; i++)
+		clear_commit_marks(tips[i], all_flags);
+	for (i = 0; i < nr_bases; i++)
+		clear_commit_marks(bases[i], all_flags);
+	if (ret) {
+		commit_list_free(*new_commits);
+		*new_commits = NULL;
+	}
+	return ret;
+}
+
 static int merge_bases_many(struct repository *r,
 			    struct commit *one, int n,
 			    struct commit **twos,

commit-reach.h

@@ -167,4 +167,18 @@ int get_branch_base_for_tip(struct repository *r,
 			    struct commit **bases,
 			    size_t bases_nr);
 
+/*
+ * Find commits reachable from 'tips' but not from 'bases' using a
+ * two-paint walk.  Requires a commit-graph with corrected commit
+ * dates (generation data v2).  Returns 0 on success with new_commits
+ * populated, or -1 to signal the caller to fall back to another
+ * method.
+ *
+ * All tips and bases must be parsed before calling.
+ */
+int repo_find_boundary_commits(struct repository *r,
+			       struct commit **bases, size_t nr_bases,
+			       size_t nr_tips, struct commit **tips,
+			       struct commit_list **new_commits);
+
 #endif