Got Mirror

Commit Diff

Commit:: e71f1e62ddab990a047fe827d13b25003a9b8014
From:: Stefan Sperling <stsp@stsp.name>
Via:: Thomas Adam <thomas@xteddy.org>
Date:: Thu Jun 23 14:09:34 2022 UTC
Message:: let got-read-pack be explicit about whether it could enumerate all objects This allows the main process to avoid looping over all object IDs again in case the pack file used for enumeration is complete. ok op@
Actions:: Patch | Tree
commit - 93088ccae4944671d38d0792e22c381cde9cc82f
commit + e71f1e62ddab990a047fe827d13b25003a9b8014
blob - 1f4430dcde3e6550f20a42cee4f1955e7ea675ba
blob + 58f6ea9d97872411048d72595be2da70a1bb58ae
--- lib/got_lib_object.h
+++ lib/got_lib_object.h
@@ -140,7 +140,7 @@ typedef const struct got_error *(*got_object_enumerate
     struct got_tree_object *, time_t, struct got_object_id *, const char *,
     struct got_repository *);
 
-const struct got_error *got_object_enumerate(got_object_enumerate_commit_cb,
-    got_object_enumerate_tree_cb, void *, struct got_object_id **, int,
-    struct got_object_id **, int, struct got_packidx *,
-    struct got_repository *);
+const struct got_error *got_object_enumerate(int *,
+    got_object_enumerate_commit_cb, got_object_enumerate_tree_cb, void *,
+    struct got_object_id **, int, struct got_object_id **, int,
+    struct got_packidx *, struct got_repository *);
blob - 7c79ce98e69f74eddb8e63cbcc86fba3f740c598
blob + 242b4340503cbbfe02b3dd739802ae8cfaf898e7
--- lib/got_lib_privsep.h
+++ lib/got_lib_privsep.h
@@ -150,6 +150,7 @@ enum got_imsg_type {
 	GOT_IMSG_ENUMERATED_TREE,
 	GOT_IMSG_TREE_ENUMERATION_DONE,
 	GOT_IMSG_OBJECT_ENUMERATION_DONE,
+	GOT_IMSG_OBJECT_ENUMERATION_INCOMPLETE,
 
 	/* Message sending file descriptor to a temporary file. */
 	GOT_IMSG_TMPFD,
@@ -748,12 +749,14 @@ const struct got_error *got_privsep_send_enumerated_tr
 const struct got_error *got_privsep_send_object_enumeration_request(
     struct imsgbuf *);
 const struct got_error *got_privsep_send_object_enumeration_done(
+    struct imsgbuf *);
+const struct got_error *got_privsep_send_object_enumeration_incomplete(
     struct imsgbuf *);
 const struct got_error *got_privsep_send_enumerated_commit(struct imsgbuf *,
     struct got_object_id *, time_t);
-const struct got_error *got_privsep_recv_enumerated_objects(struct imsgbuf *,
-    got_object_enumerate_commit_cb, got_object_enumerate_tree_cb, void *,
-    struct got_repository *);
+const struct got_error *got_privsep_recv_enumerated_objects(int *,
+    struct imsgbuf *, got_object_enumerate_commit_cb,
+    got_object_enumerate_tree_cb, void *, struct got_repository *);
 
 const struct got_error *got_privsep_send_raw_delta_req(struct imsgbuf *, int,
     struct got_object_id *);
blob - 02b89f9287f0d3749fc53d9c5b0cfe54f7d31c04
blob + d611821e6302999ac110d3b018fd2187899c7436
--- lib/object.c
+++ lib/object.c
@@ -2400,7 +2400,8 @@ done:
 }
 
 const struct got_error *
-got_object_enumerate(got_object_enumerate_commit_cb cb_commit,
+got_object_enumerate(int *found_all_objects,
+    got_object_enumerate_commit_cb cb_commit,
     got_object_enumerate_tree_cb cb_tree, void *cb_arg,
     struct got_object_id **ours, int nours,
     struct got_object_id **theirs, int ntheirs,
@@ -2449,8 +2450,8 @@ got_object_enumerate(got_object_enumerate_commit_cb cb
 	if (err)
 		goto done;
 
-	err = got_privsep_recv_enumerated_objects(pack->privsep_child->ibuf,
-	    cb_commit, cb_tree, cb_arg, repo);
+	err = got_privsep_recv_enumerated_objects(found_all_objects,
+	    pack->privsep_child->ibuf, cb_commit, cb_tree, cb_arg, repo);
 done:
 	free(path_packfile);
 	return err;
blob - 80967f22962583dccb1a9aed894ee1197af08e8a
blob + b772646b72dad6b735412e58c407030184994838
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -1563,7 +1563,8 @@ load_packed_tree_ids(void *arg, struct got_tree_object
 }
 
 static const struct got_error *
-load_packed_object_ids(struct got_object_id **ours, int nours,
+load_packed_object_ids(int *found_all_objects,
+    struct got_object_id **ours, int nours,
     struct got_object_id **theirs, int ntheirs,
     int want_meta, uint32_t seed, struct got_object_idset *idset,
     struct got_object_idset *idset_exclude, int loose_obj_only,
@@ -1591,7 +1592,7 @@ load_packed_object_ids(struct got_object_id **ours, in
 	lpa.cancel_arg = cancel_arg;
 
 	/* Attempt to load objects via got-read-pack, as far as possible. */
-	err = got_object_enumerate(load_packed_commit_id,
+	err = got_object_enumerate(found_all_objects, load_packed_commit_id,
 	   load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
 	   packidx, repo);
 	if (err)
@@ -1602,7 +1603,7 @@ load_packed_object_ids(struct got_object_id **ours, in
 
 	/*
 	 * An incomplete tree hierarchy was present in the pack file
-	 * and caused loading to be aborted midway through a commit.
+	 * and caused loading to be aborted.
 	 * Continue loading trees the slow way.
 	 */
 	err = load_tree(want_meta, idset, idset_exclude,
@@ -1674,7 +1675,7 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 	const struct got_error *err = NULL;
 	struct got_object_id **ids = NULL;
 	struct got_packidx *packidx = NULL;
-	int i, nobj = 0, obj_type;
+	int i, nobj = 0, obj_type, found_all_objects = 0;
 	struct got_object_idset *idset_exclude;
 
 	idset_exclude = got_object_idset_alloc();
@@ -1694,10 +1695,10 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 	if (err)
 		goto done;
 	if (packidx) {
-		err = load_packed_object_ids(theirs, ntheirs, NULL, 0, 0,
-		    seed, idset, idset_exclude, loose_obj_only, repo, packidx,
-		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
-		    cancel_cb, cancel_arg);
+		err = load_packed_object_ids(&found_all_objects,
+		    theirs, ntheirs, NULL, 0, 0, seed, idset, idset_exclude,
+		    loose_obj_only, repo, packidx, ncolored, nfound, ntrees,
+		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
@@ -1710,12 +1711,15 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 		if (err)
 			return err;
 		if (obj_type == GOT_OBJ_TYPE_COMMIT) {
-			err = load_commit(0, idset, idset_exclude, id, repo,
-			    seed, loose_obj_only, ncolored, nfound, ntrees,
-			    progress_cb, progress_arg, rl,
-			    cancel_cb, cancel_arg);
-			if (err)
-				goto done;
+			if (!found_all_objects) {
+				err = load_commit(0, idset, idset_exclude,
+				    id, repo, seed, loose_obj_only,
+				    ncolored, nfound, ntrees,
+				    progress_cb, progress_arg, rl,
+				    cancel_cb, cancel_arg);
+				if (err)
+					goto done;
+			}
 		} else if (obj_type == GOT_OBJ_TYPE_TAG) {
 			err = load_tag(0, idset, idset_exclude, id, repo,
 			    seed, loose_obj_only, ncolored, nfound, ntrees,
@@ -1726,24 +1730,28 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 		}
 	}
 
+	found_all_objects = 0;
 	err = find_pack_for_enumeration(&packidx, ids, nobj, repo);
 	if (err)
 		goto done;
 	if (packidx) {
-		err = load_packed_object_ids(ids, nobj, theirs, ntheirs, 1,
-		    seed, idset, idset_exclude, loose_obj_only, repo, packidx,
-		    ncolored, nfound, ntrees,
+		err = load_packed_object_ids(&found_all_objects, ids,
+		    nobj, theirs, ntheirs, 1, seed, idset, idset_exclude,
+		    loose_obj_only, repo, packidx, ncolored, nfound, ntrees,
 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
 
-	for (i = 0; i < nobj; i++) {
-		err = load_commit(1, idset, idset_exclude, ids[i], repo,
-		    seed, loose_obj_only, ncolored, nfound, ntrees,
-		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
-		if (err)
-			goto done;
+	if (!found_all_objects) {
+		for (i = 0; i < nobj; i++) {
+			err = load_commit(1, idset, idset_exclude, ids[i],
+			    repo, seed, loose_obj_only, ncolored, nfound,
+			    ntrees, progress_cb, progress_arg, rl,
+			    cancel_cb, cancel_arg);
+			if (err)
+				goto done;
+		}
 	}
 
 	for (i = 0; i < nours; i++) {
blob - 2d6613e89e286a3d0a92d772b04ae4182aad14ca
blob + f6f70cc0eaf2082147dd6cfcf7da7c1a010486f9
--- lib/privsep.c
+++ lib/privsep.c
@@ -2750,6 +2750,17 @@ got_privsep_send_object_enumeration_done(struct imsgbu
 }
 
 const struct got_error *
+got_privsep_send_object_enumeration_incomplete(struct imsgbuf *ibuf)
+{
+	if (imsg_compose(ibuf, GOT_IMSG_OBJECT_ENUMERATION_INCOMPLETE,
+	    0, 0, -1, NULL, 0) == -1)
+		return got_error_from_errno("imsg_compose "
+		    "OBJECT_ENUMERATION_INCOMPLETE");
+
+	return flush_imsg(ibuf);
+}
+
+const struct got_error *
 got_privsep_send_enumerated_commit(struct imsgbuf *ibuf,
     struct got_object_id *id, time_t mtime)
 {
@@ -2773,7 +2784,8 @@ got_privsep_send_enumerated_commit(struct imsgbuf *ibu
 }
 
 const struct got_error *
-got_privsep_recv_enumerated_objects(struct imsgbuf *ibuf,
+got_privsep_recv_enumerated_objects(int *found_all_objects,
+    struct imsgbuf *ibuf,
     got_object_enumerate_commit_cb cb_commit,
     got_object_enumerate_tree_cb cb_tree, void *cb_arg,
     struct got_repository *repo)
@@ -2792,6 +2804,7 @@ got_privsep_recv_enumerated_objects(struct imsgbuf *ib
 	int nentries = -1;
 	int done = 0;
 
+	*found_all_objects = 0;
 	memset(&tree, 0, sizeof(tree));
 
 	while (!done) {
@@ -2939,6 +2952,10 @@ got_privsep_recv_enumerated_objects(struct imsgbuf *ib
 			have_commit = 0;
 			break;
 		case GOT_IMSG_OBJECT_ENUMERATION_DONE:
+			*found_all_objects = 1;
+			done = 1;
+			break;
+		case GOT_IMSG_OBJECT_ENUMERATION_INCOMPLETE:
 			done = 1;
 			break;
 		default:
blob - b38dbc844d8cdf257a5d3a9b38a75ad4955dc287
blob + ccb6ab4e634be903e2e13bc07de7e337b68652ad
--- libexec/got-read-pack/got-read-pack.c
+++ libexec/got-read-pack/got-read-pack.c
@@ -1407,8 +1407,10 @@ enumeration_request(struct imsg *imsg, struct imsgbuf 
 		}
 
 		idx = got_packidx_get_object_idx(packidx, &qid->id);
-		if (idx == -1)
+		if (idx == -1) {
+			have_all_entries = 0;
 			break;
+		}
 
 		err = open_object(&obj, pack, packidx, idx, &qid->id,
 		    objcache);
@@ -1429,8 +1431,10 @@ enumeration_request(struct imsg *imsg, struct imsgbuf 
 				goto done;
 			}
 			idx = got_packidx_get_object_idx(packidx, &tag->id);
-			if (idx == -1)
+			if (idx == -1) {
+				have_all_entries = 0;
 				break;
+			}
 			err = open_commit(&commit, pack, packidx, idx,
 			    &tag->id, objcache);
 			got_object_tag_close(tag);
@@ -1457,6 +1461,7 @@ enumeration_request(struct imsg *imsg, struct imsgbuf 
 		tree_id = got_object_commit_get_tree_id(commit);
 		idx = got_packidx_get_object_idx(packidx, tree_id);
 		if (idx == -1) {
+			have_all_entries = 0;
 			err = got_privsep_send_enumerated_tree(&totlen, ibuf,
 			    tree_id, "/", NULL, -1);
 			if (err)
@@ -1502,6 +1507,10 @@ enumeration_request(struct imsg *imsg, struct imsgbuf 
 
 	if (have_all_entries) {
 		err = got_privsep_send_object_enumeration_done(ibuf);
+		if (err)
+			goto done;
+	} else {
+		err = got_privsep_send_object_enumeration_incomplete(ibuf);
 		if (err)
 			goto done;
 	}