Got Mirror

Blob

Date:: Sat May 7 11:50:56 2022 UTC
Message:: run the search for deltas to reuse in got-read-pack This significantly speeds up the deltification step of packing by avoiding imsg traffic. gotadmin no longer requests individual raw deltas from got-read-pack to check whether it can reuse them. Instead, got-read-pack obtains a list of objects we want to pack, and hands back the list of all deltas in its pack file which can be reused. Messages are now batched such that imsg buffers are filled as much as possible. Another advantage is that deltas we are not going to reuse will no longer be written to the delta cache file, saving disk space. Before this patch, any raw delta candidate was written to the delta cache file by got-read-pack, and the decision whether to reuse the delta happened afterwards in the gotadmin process. Code for reading individual raw deltas is now unused and could be removed at some point. ok op@
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/uio.h>
22 #include <sys/socket.h>
23 #include <sys/wait.h>
24 #include <sys/resource.h>
25 #include <sys/mman.h>
26 
27 #include <errno.h>
28 #include <fcntl.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <stdint.h>
33 #include <sha1.h>
34 #include <unistd.h>
35 #include <zlib.h>
36 #include <ctype.h>
37 #include <libgen.h>
38 #include <limits.h>
39 #include <imsg.h>
40 #include <time.h>
41 
42 #include "got_error.h"
43 #include "got_object.h"
44 #include "got_repository.h"
45 #include "got_opentemp.h"
46 #include "got_path.h"
47 
48 #include "got_lib_sha1.h"
49 #include "got_lib_delta.h"
50 #include "got_lib_inflate.h"
51 #include "got_lib_object.h"
52 #include "got_lib_privsep.h"
53 #include "got_lib_object_idcache.h"
54 #include "got_lib_object_cache.h"
55 #include "got_lib_object_parse.h"
56 #include "got_lib_pack.h"
57 #include "got_lib_repository.h"
58 
59 #ifndef MIN
60 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
61 #endif
62 
63 struct got_object_id *
64 got_object_get_id(struct got_object *obj)
65 {
66 	return &obj->id;
67 }
68 
69 const struct got_error *
70 got_object_get_id_str(char **outbuf, struct got_object *obj)
71 {
72 	return got_object_id_str(outbuf, &obj->id);
73 }
74 
75 const struct got_error *
76 got_object_get_type(int *type, struct got_repository *repo,
77     struct got_object_id *id)
78 {
79 	const struct got_error *err = NULL;
80 	struct got_object *obj;
81 
82 	err = got_object_open(&obj, repo, id);
83 	if (err)
84 		return err;
85 
86 	switch (obj->type) {
87 	case GOT_OBJ_TYPE_COMMIT:
88 	case GOT_OBJ_TYPE_TREE:
89 	case GOT_OBJ_TYPE_BLOB:
90 	case GOT_OBJ_TYPE_TAG:
91 		*type = obj->type;
92 		break;
93 	default:
94 		err = got_error(GOT_ERR_OBJ_TYPE);
95 		break;
96 	}
97 
98 	got_object_close(obj);
99 	return err;
100 }
101 
102 const struct got_error *
103 got_object_get_path(char **path, struct got_object_id *id,
104     struct got_repository *repo)
105 {
106 	const struct got_error *err = NULL;
107 	char *hex = NULL;
108 	char *path_objects;
109 
110 	*path = NULL;
111 
112 	path_objects = got_repo_get_path_objects(repo);
113 	if (path_objects == NULL)
114 		return got_error_from_errno("got_repo_get_path_objects");
115 
116 	err = got_object_id_str(&hex, id);
117 	if (err)
118 		goto done;
119 
120 	if (asprintf(path, "%s/%.2x/%s", path_objects,
121 	    id->sha1[0], hex + 2) == -1)
122 		err = got_error_from_errno("asprintf");
123 
124 done:
125 	free(hex);
126 	free(path_objects);
127 	return err;
128 }
129 
130 const struct got_error *
131 got_object_open_loose_fd(int *fd, struct got_object_id *id,
132     struct got_repository *repo)
133 {
134 	const struct got_error *err = NULL;
135 	char *path;
136 
137 	err = got_object_get_path(&path, id, repo);
138 	if (err)
139 		return err;
140 	*fd = open(path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
141 	if (*fd == -1) {
142 		err = got_error_from_errno2("open", path);
143 		goto done;
144 	}
145 done:
146 	free(path);
147 	return err;
148 }
149 
150 static const struct got_error *
151 request_packed_object(struct got_object **obj, struct got_pack *pack, int idx,
152     struct got_object_id *id)
153 {
154 	const struct got_error *err = NULL;
155 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
156 
157 	err = got_privsep_send_packed_obj_req(ibuf, idx, id);
158 	if (err)
159 		return err;
160 
161 	err = got_privsep_recv_obj(obj, ibuf);
162 	if (err)
163 		return err;
164 
165 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
166 
167 	return NULL;
168 }
169 
170 /* Create temporary files used during delta application. */
171 static const struct got_error *
172 pack_child_send_tempfiles(struct imsgbuf *ibuf, struct got_pack *pack)
173 {
174 	const struct got_error *err;
175 	int basefd, accumfd;
176 
177 	/* 
178 	 * For performance reasons, the child will keep reusing the
179 	 * same temporary files during every object request.
180 	 * Opening and closing new files for every object request is
181 	 * too expensive during operations such as 'gotadmin pack'.
182 	 */
183 	if (pack->child_has_tempfiles)
184 		return NULL;
185 
186 	basefd = got_opentempfd();
187 	if (basefd == -1)
188 		return got_error_from_errno("got_opentempfd");
189 
190 	err = got_privsep_send_tmpfd(ibuf, basefd);
191 	if (err)
192 		return err;
193 
194 	accumfd = got_opentempfd();
195 	if (accumfd == -1)
196 		return got_error_from_errno("got_opentempfd");
197 
198 	err = got_privsep_send_tmpfd(ibuf, accumfd);
199 	if (err)
200 		return err;
201 
202 	pack->child_has_tempfiles = 1;
203 	return NULL;
204 }
205 
206 static const struct got_error *
207 request_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen,
208     int outfd, struct got_pack *pack, int idx, struct got_object_id *id)
209 {
210 	const struct got_error *err = NULL;
211 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
212 	int outfd_child;
213 
214 	 err = pack_child_send_tempfiles(ibuf, pack);
215 	 if (err)
216 		return err;
217 
218 	outfd_child = dup(outfd);
219 	if (outfd_child == -1)
220 		return got_error_from_errno("dup");
221 
222 	err = got_privsep_send_packed_raw_obj_req(ibuf, idx, id);
223 	if (err) {
224 		close(outfd_child);
225 		return err;
226 	}
227 
228 	err = got_privsep_send_raw_obj_outfd(ibuf, outfd_child);
229 	if (err)
230 		return err;
231 
232 	err = got_privsep_recv_raw_obj(outbuf, size, hdrlen, ibuf);
233 	if (err)
234 		return err;
235 
236 	return NULL;
237 }
238 
239 static void
240 set_max_datasize(void)
241 {
242 	struct rlimit rl;
243 
244 	if (getrlimit(RLIMIT_DATA, &rl) != 0)
245 		return;
246 
247 	rl.rlim_cur = rl.rlim_max;
248 	setrlimit(RLIMIT_DATA, &rl);
249 }
250 
251 static const struct got_error *
252 start_pack_privsep_child(struct got_pack *pack, struct got_packidx *packidx)
253 {
254 	const struct got_error *err = NULL;
255 	int imsg_fds[2];
256 	pid_t pid;
257 	struct imsgbuf *ibuf;
258 
259 	ibuf = calloc(1, sizeof(*ibuf));
260 	if (ibuf == NULL)
261 		return got_error_from_errno("calloc");
262 
263 	pack->privsep_child = calloc(1, sizeof(*pack->privsep_child));
264 	if (pack->privsep_child == NULL) {
265 		err = got_error_from_errno("calloc");
266 		free(ibuf);
267 		return err;
268 	}
269 	pack->child_has_tempfiles = 0;
270 	pack->child_has_delta_outfd = 0;
271 
272 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
273 		err = got_error_from_errno("socketpair");
274 		goto done;
275 	}
276 
277 	pid = fork();
278 	if (pid == -1) {
279 		err = got_error_from_errno("fork");
280 		goto done;
281 	} else if (pid == 0) {
282 		set_max_datasize();
283 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_PACK,
284 		    pack->path_packfile);
285 		/* not reached */
286 	}
287 
288 	if (close(imsg_fds[1]) == -1)
289 		return got_error_from_errno("close");
290 	pack->privsep_child->imsg_fd = imsg_fds[0];
291 	pack->privsep_child->pid = pid;
292 	imsg_init(ibuf, imsg_fds[0]);
293 	pack->privsep_child->ibuf = ibuf;
294 
295 	err = got_privsep_init_pack_child(ibuf, pack, packidx);
296 	if (err) {
297 		const struct got_error *child_err;
298 		err = got_privsep_send_stop(pack->privsep_child->imsg_fd);
299 		child_err = got_privsep_wait_for_child(
300 		    pack->privsep_child->pid);
301 		if (child_err && err == NULL)
302 			err = child_err;
303 	}
304 done:
305 	if (err) {
306 		free(ibuf);
307 		free(pack->privsep_child);
308 		pack->privsep_child = NULL;
309 	}
310 	return err;
311 }
312 
313 static const struct got_error *
314 read_packed_object_privsep(struct got_object **obj,
315     struct got_repository *repo, struct got_pack *pack,
316     struct got_packidx *packidx, int idx, struct got_object_id *id)
317 {
318 	const struct got_error *err = NULL;
319 
320 	if (pack->privsep_child == NULL) {
321 		err = start_pack_privsep_child(pack, packidx);
322 		if (err)
323 			return err;
324 	}
325 
326 	return request_packed_object(obj, pack, idx, id);
327 }
328 
329 static const struct got_error *
330 read_packed_object_raw_privsep(uint8_t **outbuf, off_t *size, size_t *hdrlen,
331     int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
332     struct got_object_id *id)
333 {
334 	const struct got_error *err = NULL;
335 
336 	if (pack->privsep_child == NULL) {
337 		err = start_pack_privsep_child(pack, packidx);
338 		if (err)
339 			return err;
340 	}
341 
342 	return request_packed_object_raw(outbuf, size, hdrlen, outfd, pack,
343 	    idx, id);
344 }
345 
346 const struct got_error *
347 got_object_open_packed(struct got_object **obj, struct got_object_id *id,
348     struct got_repository *repo)
349 {
350 	const struct got_error *err = NULL;
351 	struct got_pack *pack = NULL;
352 	struct got_packidx *packidx = NULL;
353 	int idx;
354 	char *path_packfile;
355 
356 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
357 	if (err)
358 		return err;
359 
360 	err = got_packidx_get_packfile_path(&path_packfile,
361 	    packidx->path_packidx);
362 	if (err)
363 		return err;
364 
365 	pack = got_repo_get_cached_pack(repo, path_packfile);
366 	if (pack == NULL) {
367 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
368 		if (err)
369 			goto done;
370 	}
371 
372 	err = read_packed_object_privsep(obj, repo, pack, packidx, idx, id);
373 	if (err)
374 		goto done;
375 done:
376 	free(path_packfile);
377 	return err;
378 }
379 
380 const struct got_error *
381 got_object_open_from_packfile(struct got_object **obj, struct got_object_id *id,
382     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
383     struct got_repository *repo)
384 {
385 	return read_packed_object_privsep(obj, repo, pack, packidx,
386 	    obj_idx, id);
387 }
388 
389 const struct got_error *
390 got_object_read_raw_delta(uint64_t *base_size, uint64_t *result_size,
391     off_t *delta_size, off_t *delta_compressed_size, off_t *delta_offset,
392     off_t *delta_out_offset, struct got_object_id **base_id, int delta_cache_fd,
393     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
394     struct got_repository *repo)
395 {
396 	const struct got_error *err = NULL;
397 	struct got_pack *pack = NULL;
398 	char *path_packfile;
399 
400 	*base_size = 0;
401 	*result_size = 0;
402 	*delta_size = 0;
403 	*delta_compressed_size = 0;
404 	*delta_offset = 0;
405 	*delta_out_offset = 0;
406 
407 	err = got_packidx_get_packfile_path(&path_packfile,
408 	    packidx->path_packidx);
409 	if (err)
410 		return err;
411 
412 	pack = got_repo_get_cached_pack(repo, path_packfile);
413 	if (pack == NULL) {
414 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
415 		if (err)
416 			return err;
417 	}
418 
419 	if (pack->privsep_child == NULL) {
420 		err = start_pack_privsep_child(pack, packidx);
421 		if (err)
422 			return err;
423 	}
424 
425 	if (!pack->child_has_delta_outfd) {
426 		int outfd_child;
427 		outfd_child = dup(delta_cache_fd);
428 		if (outfd_child == -1)
429 			return got_error_from_errno("dup");
430 		err = got_privsep_send_raw_delta_outfd(
431 		    pack->privsep_child->ibuf, outfd_child);
432 		if (err)
433 			return err;
434 		pack->child_has_delta_outfd = 1;
435 	}
436 
437 	err = got_privsep_send_raw_delta_req(pack->privsep_child->ibuf,
438 	    obj_idx, id);
439 	if (err)
440 		return err;
441 
442 	return got_privsep_recv_raw_delta(base_size, result_size, delta_size,
443 	    delta_compressed_size, delta_offset, delta_out_offset, base_id,
444 	    pack->privsep_child->ibuf);
445 }
446 
447 /*
448  * XXX This function does not really belong in object.c. It is only here
449  * because it needs start_pack_privsep_child(); relevant code should
450  * probably be moved to pack.c/pack_create.c.
451  */
452 const struct got_error *
453 got_object_prepare_delta_reuse(struct got_pack **pack,
454     struct got_packidx *packidx, int delta_outfd, struct got_repository *repo)
455 {
456 	const struct got_error *err = NULL;
457 	char *path_packfile = NULL;
458 
459 	err = got_packidx_get_packfile_path(&path_packfile,
460 	    packidx->path_packidx);
461 	if (err)
462 		return err;
463 
464 	*pack = got_repo_get_cached_pack(repo, path_packfile);
465 	if (*pack == NULL) {
466 		err = got_repo_cache_pack(pack, repo, path_packfile, packidx);
467 		if (err)
468 			goto done;
469 	}
470 	if ((*pack)->privsep_child == NULL) {
471 		err = start_pack_privsep_child(*pack, packidx);
472 		if (err)
473 			goto done;
474 	}
475 
476 	if (!(*pack)->child_has_delta_outfd) {
477 		int outfd_child;
478 		outfd_child = dup(delta_outfd);
479 		if (outfd_child == -1) {
480 			err = got_error_from_errno("dup");
481 			goto done;
482 		}
483 		err = got_privsep_send_raw_delta_outfd(
484 		    (*pack)->privsep_child->ibuf, outfd_child);
485 		if (err)
486 			goto done;
487 		(*pack)->child_has_delta_outfd = 1;
488 	}
489 
490 	err = got_privsep_send_delta_reuse_req((*pack)->privsep_child->ibuf);
491 done:
492 	free(path_packfile);
493 	return err;
494 }
495 
496 static const struct got_error *
497 request_object(struct got_object **obj, struct got_object_id *id,
498     struct got_repository *repo, int fd)
499 {
500 	const struct got_error *err = NULL;
501 	struct imsgbuf *ibuf;
502 
503 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf;
504 
505 	err = got_privsep_send_obj_req(ibuf, fd, id);
506 	if (err)
507 		return err;
508 
509 	return got_privsep_recv_obj(obj, ibuf);
510 }
511 
512 static const struct got_error *
513 request_raw_object(uint8_t **outbuf, off_t *size, size_t *hdrlen, int outfd,
514     struct got_object_id *id, struct got_repository *repo, int infd)
515 {
516 	const struct got_error *err = NULL;
517 	struct imsgbuf *ibuf;
518 	int outfd_child;
519 
520 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf;
521 
522 	outfd_child = dup(outfd);
523 	if (outfd_child == -1)
524 		return got_error_from_errno("dup");
525 
526 	err = got_privsep_send_raw_obj_req(ibuf, infd, id);
527 	if (err)
528 		return err;
529 
530 	err = got_privsep_send_raw_obj_outfd(ibuf, outfd_child);
531 	if (err)
532 		return err;
533 
534 	return got_privsep_recv_raw_obj(outbuf, size, hdrlen, ibuf);
535 }
536 
537 static const struct got_error *
538 start_read_object_child(struct got_repository *repo)
539 {
540 	const struct got_error *err = NULL;
541 	int imsg_fds[2];
542 	pid_t pid;
543 	struct imsgbuf *ibuf;
544 
545 	ibuf = calloc(1, sizeof(*ibuf));
546 	if (ibuf == NULL)
547 		return got_error_from_errno("calloc");
548 
549 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
550 		err = got_error_from_errno("socketpair");
551 		free(ibuf);
552 		return err;
553 	}
554 
555 	pid = fork();
556 	if (pid == -1) {
557 		err = got_error_from_errno("fork");
558 		free(ibuf);
559 		return err;
560 	}
561 	else if (pid == 0) {
562 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_OBJECT,
563 		    repo->path);
564 		/* not reached */
565 	}
566 
567 	if (close(imsg_fds[1]) == -1) {
568 		err = got_error_from_errno("close");
569 		free(ibuf);
570 		return err;
571 	}
572 
573 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd =
574 	    imsg_fds[0];
575 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].pid = pid;
576 	imsg_init(ibuf, imsg_fds[0]);
577 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].ibuf = ibuf;
578 
579 	return NULL;
580 }
581 
582 const struct got_error *
583 got_object_read_header_privsep(struct got_object **obj,
584     struct got_object_id *id, struct got_repository *repo, int obj_fd)
585 {
586 	const struct got_error *err;
587 
588 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd != -1)
589 		return request_object(obj, id, repo, obj_fd);
590 
591 	err = start_read_object_child(repo);
592 	if (err) {
593 		close(obj_fd);
594 		return err;
595 	}
596 
597 	return request_object(obj, id, repo, obj_fd);
598 }
599 
600 static const struct got_error *
601 read_object_raw_privsep(uint8_t **outbuf, off_t *size, size_t *hdrlen,
602     int outfd, struct got_object_id *id, struct got_repository *repo,
603     int obj_fd)
604 {
605 	const struct got_error *err;
606 
607 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_OBJECT].imsg_fd != -1)
608 		return request_raw_object(outbuf, size, hdrlen, outfd, id,
609 		    repo, obj_fd);
610 
611 	err = start_read_object_child(repo);
612 	if (err)
613 		return err;
614 
615 	return request_raw_object(outbuf, size, hdrlen, outfd, id, repo,
616 	    obj_fd);
617 }
618 
619 const struct got_error *
620 got_object_open(struct got_object **obj, struct got_repository *repo,
621     struct got_object_id *id)
622 {
623 	const struct got_error *err = NULL;
624 	int fd;
625 
626 	*obj = got_repo_get_cached_object(repo, id);
627 	if (*obj != NULL) {
628 		(*obj)->refcnt++;
629 		return NULL;
630 	}
631 
632 	err = got_object_open_packed(obj, id, repo);
633 	if (err && err->code != GOT_ERR_NO_OBJ)
634 		return err;
635 	if (*obj) {
636 		(*obj)->refcnt++;
637 		return got_repo_cache_object(repo, id, *obj);
638 	}
639 
640 	err = got_object_open_loose_fd(&fd, id, repo);
641 	if (err) {
642 		if (err->code == GOT_ERR_ERRNO && errno == ENOENT)
643 			err = got_error_no_obj(id);
644 		return err;
645 	}
646 
647 	err = got_object_read_header_privsep(obj, id, repo, fd);
648 	if (err)
649 		return err;
650 
651 	memcpy((*obj)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH);
652 
653 	(*obj)->refcnt++;
654 	return got_repo_cache_object(repo, id, *obj);
655 }
656 
657 /* *outfd must be initialized to -1 by caller */
658 const struct got_error *
659 got_object_raw_open(struct got_raw_object **obj, int *outfd,
660     struct got_repository *repo, struct got_object_id *id)
661 {
662 	const struct got_error *err = NULL;
663 	struct got_packidx *packidx = NULL;
664 	int idx;
665 	uint8_t *outbuf = NULL;
666 	off_t size = 0;
667 	size_t hdrlen = 0;
668 	char *path_packfile = NULL;
669 
670 	*obj = got_repo_get_cached_raw_object(repo, id);
671 	if (*obj != NULL) {
672 		(*obj)->refcnt++;
673 		return NULL;
674 	}
675 
676 	if (*outfd == -1) {
677 		*outfd = got_opentempfd();
678 		if (*outfd == -1)
679 			return got_error_from_errno("got_opentempfd");
680 	}
681 
682 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
683 	if (err == NULL) {
684 		struct got_pack *pack = NULL;
685 
686 		err = got_packidx_get_packfile_path(&path_packfile,
687 		    packidx->path_packidx);
688 		if (err)
689 			goto done;
690 
691 		pack = got_repo_get_cached_pack(repo, path_packfile);
692 		if (pack == NULL) {
693 			err = got_repo_cache_pack(&pack, repo, path_packfile,
694 			    packidx);
695 			if (err)
696 				goto done;
697 		}
698 		err = read_packed_object_raw_privsep(&outbuf, &size, &hdrlen,
699 		    *outfd, pack, packidx, idx, id);
700 		if (err)
701 			goto done;
702 	} else if (err->code == GOT_ERR_NO_OBJ) {
703 		int fd;
704 
705 		err = got_object_open_loose_fd(&fd, id, repo);
706 		if (err)
707 			goto done;
708 		err = read_object_raw_privsep(&outbuf, &size, &hdrlen, *outfd,
709 		    id, repo, fd);
710 		if (err)
711 			goto done;
712 	}
713 
714 	*obj = calloc(1, sizeof(**obj));
715 	if (*obj == NULL) {
716 		err = got_error_from_errno("calloc");
717 		goto done;
718 	}
719 	(*obj)->fd = -1;
720 
721 	if (outbuf) {
722 		(*obj)->data = outbuf;
723 	} else {
724 		struct stat sb;
725 		if (fstat(*outfd, &sb) == -1) {
726 			err = got_error_from_errno("fstat");
727 			goto done;
728 		}
729 
730 		if (sb.st_size != hdrlen + size) {
731 			err = got_error(GOT_ERR_PRIVSEP_LEN);
732 			goto done;
733 		}
734 #ifndef GOT_PACK_NO_MMAP
735 		if (hdrlen + size > 0) {
736 			(*obj)->data = mmap(NULL, hdrlen + size, PROT_READ,
737 			    MAP_PRIVATE, *outfd, 0);
738 			if ((*obj)->data == MAP_FAILED) {
739 				if (errno != ENOMEM) {
740 					err = got_error_from_errno("mmap");
741 					goto done;
742 				}
743 				(*obj)->data = NULL;
744 			} else {
745 				(*obj)->fd = *outfd;
746 				*outfd = -1;
747 			}
748 		}
749 #endif
750 		if (*outfd != -1) {
751 			(*obj)->f = fdopen(*outfd, "r");
752 			if ((*obj)->f == NULL) {
753 				err = got_error_from_errno("fdopen");
754 				goto done;
755 			}
756 			*outfd = -1;
757 		}
758 	}
759 	(*obj)->hdrlen = hdrlen;
760 	(*obj)->size = size;
761 	err = got_repo_cache_raw_object(repo, id, *obj);
762 done:
763 	free(path_packfile);
764 	if (err) {
765 		if (*obj) {
766 			got_object_raw_close(*obj);
767 			*obj = NULL;
768 		}
769 		free(outbuf);
770 	} else
771 		(*obj)->refcnt++;
772 	return err;
773 }
774 
775 const struct got_error *
776 got_object_open_by_id_str(struct got_object **obj, struct got_repository *repo,
777     const char *id_str)
778 {
779 	struct got_object_id id;
780 
781 	if (!got_parse_sha1_digest(id.sha1, id_str))
782 		return got_error_path(id_str, GOT_ERR_BAD_OBJ_ID_STR);
783 
784 	return got_object_open(obj, repo, &id);
785 }
786 
787 const struct got_error *
788 got_object_resolve_id_str(struct got_object_id **id,
789     struct got_repository *repo, const char *id_str)
790 {
791 	const struct got_error *err = NULL;
792 	struct got_object *obj;
793 
794 	err = got_object_open_by_id_str(&obj, repo, id_str);
795 	if (err)
796 		return err;
797 
798 	*id = got_object_id_dup(got_object_get_id(obj));
799 	got_object_close(obj);
800 	if (*id == NULL)
801 		return got_error_from_errno("got_object_id_dup");
802 
803 	return NULL;
804 }
805 
806 static const struct got_error *
807 request_packed_commit(struct got_commit_object **commit, struct got_pack *pack,
808     int pack_idx, struct got_object_id *id)
809 {
810 	const struct got_error *err = NULL;
811 
812 	err = got_privsep_send_commit_req(pack->privsep_child->ibuf, -1, id,
813 	    pack_idx);
814 	if (err)
815 		return err;
816 
817 	err = got_privsep_recv_commit(commit, pack->privsep_child->ibuf);
818 	if (err)
819 		return err;
820 
821 	(*commit)->flags |= GOT_COMMIT_FLAG_PACKED;
822 	return NULL;
823 }
824 
825 static const struct got_error *
826 read_packed_commit_privsep(struct got_commit_object **commit,
827     struct got_pack *pack, struct got_packidx *packidx, int idx,
828     struct got_object_id *id)
829 {
830 	const struct got_error *err = NULL;
831 
832 	if (pack->privsep_child)
833 		return request_packed_commit(commit, pack, idx, id);
834 
835 	err = start_pack_privsep_child(pack, packidx);
836 	if (err)
837 		return err;
838 
839 	return request_packed_commit(commit, pack, idx, id);
840 }
841 
842 static const struct got_error *
843 request_commit(struct got_commit_object **commit, struct got_repository *repo,
844     int fd, struct got_object_id *id)
845 {
846 	const struct got_error *err = NULL;
847 	struct imsgbuf *ibuf;
848 
849 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf;
850 
851 	err = got_privsep_send_commit_req(ibuf, fd, id, -1);
852 	if (err)
853 		return err;
854 
855 	return got_privsep_recv_commit(commit, ibuf);
856 }
857 
858 static const struct got_error *
859 read_commit_privsep(struct got_commit_object **commit, int obj_fd,
860     struct got_object_id *id, struct got_repository *repo)
861 {
862 	const struct got_error *err;
863 	int imsg_fds[2];
864 	pid_t pid;
865 	struct imsgbuf *ibuf;
866 
867 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].imsg_fd != -1)
868 		return request_commit(commit, repo, obj_fd, id);
869 
870 	ibuf = calloc(1, sizeof(*ibuf));
871 	if (ibuf == NULL)
872 		return got_error_from_errno("calloc");
873 
874 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
875 		err = got_error_from_errno("socketpair");
876 		free(ibuf);
877 		return err;
878 	}
879 
880 	pid = fork();
881 	if (pid == -1) {
882 		err = got_error_from_errno("fork");
883 		free(ibuf);
884 		return err;
885 	}
886 	else if (pid == 0) {
887 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_COMMIT,
888 		    repo->path);
889 		/* not reached */
890 	}
891 
892 	if (close(imsg_fds[1]) == -1) {
893 		err = got_error_from_errno("close");
894 		free(ibuf);
895 		return err;
896 	}
897 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].imsg_fd =
898 	    imsg_fds[0];
899 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].pid = pid;
900 	imsg_init(ibuf, imsg_fds[0]);
901 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf = ibuf;
902 
903 	return request_commit(commit, repo, obj_fd, id);
904 }
905 
906 
907 static const struct got_error *
908 open_commit(struct got_commit_object **commit,
909     struct got_repository *repo, struct got_object_id *id, int check_cache)
910 {
911 	const struct got_error *err = NULL;
912 	struct got_packidx *packidx = NULL;
913 	int idx;
914 	char *path_packfile = NULL;
915 
916 	if (check_cache) {
917 		*commit = got_repo_get_cached_commit(repo, id);
918 		if (*commit != NULL) {
919 			(*commit)->refcnt++;
920 			return NULL;
921 		}
922 	} else
923 		*commit = NULL;
924 
925 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
926 	if (err == NULL) {
927 		struct got_pack *pack = NULL;
928 
929 		err = got_packidx_get_packfile_path(&path_packfile,
930 		    packidx->path_packidx);
931 		if (err)
932 			return err;
933 
934 		pack = got_repo_get_cached_pack(repo, path_packfile);
935 		if (pack == NULL) {
936 			err = got_repo_cache_pack(&pack, repo, path_packfile,
937 			    packidx);
938 			if (err)
939 				goto done;
940 		}
941 		err = read_packed_commit_privsep(commit, pack,
942 		    packidx, idx, id);
943 	} else if (err->code == GOT_ERR_NO_OBJ) {
944 		int fd;
945 
946 		err = got_object_open_loose_fd(&fd, id, repo);
947 		if (err)
948 			return err;
949 		err = read_commit_privsep(commit, fd, id, repo);
950 	}
951 
952 	if (err == NULL) {
953 		(*commit)->refcnt++;
954 		err = got_repo_cache_commit(repo, id, *commit);
955 	}
956 done:
957 	free(path_packfile);
958 	return err;
959 }
960 
961 const struct got_error *
962 got_object_open_as_commit(struct got_commit_object **commit,
963     struct got_repository *repo, struct got_object_id *id)
964 {
965 	*commit = got_repo_get_cached_commit(repo, id);
966 	if (*commit != NULL) {
967 		(*commit)->refcnt++;
968 		return NULL;
969 	}
970 
971 	return open_commit(commit, repo, id, 0);
972 }
973 
974 const struct got_error *
975 got_object_commit_open(struct got_commit_object **commit,
976     struct got_repository *repo, struct got_object *obj)
977 {
978 	return open_commit(commit, repo, got_object_get_id(obj), 1);
979 }
980 
981 const struct got_error *
982 got_object_qid_alloc(struct got_object_qid **qid, struct got_object_id *id)
983 {
984 	*qid = calloc(1, sizeof(**qid));
985 	if (*qid == NULL)
986 		return got_error_from_errno("calloc");
987 
988 	memcpy(&(*qid)->id, id, sizeof((*qid)->id));
989 	return NULL;
990 }
991 
992 const struct got_error *
993 got_object_id_queue_copy(const struct got_object_id_queue *src,
994     struct got_object_id_queue *dest)
995 {
996 	const struct got_error *err;
997 	struct got_object_qid *qid;
998 
999 	STAILQ_FOREACH(qid, src, entry) {
1000 		struct got_object_qid *new;
1001 		/*
1002 		 * Deep-copy the object ID only. Let the caller deal
1003 		 * with setting up the new->data pointer if needed.
1004 		 */
1005 		err = got_object_qid_alloc(&new, &qid->id); 
1006 		if (err) {
1007 			got_object_id_queue_free(dest);
1008 			return err;
1009 		}
1010 		STAILQ_INSERT_TAIL(dest, new, entry);
1011 	}
1012 
1013 	return NULL;
1014 }
1015 
1016 static const struct got_error *
1017 request_packed_tree(struct got_tree_object **tree, struct got_pack *pack,
1018     int pack_idx, struct got_object_id *id)
1019 {
1020 	const struct got_error *err = NULL;
1021 
1022 	err = got_privsep_send_tree_req(pack->privsep_child->ibuf, -1, id,
1023 	    pack_idx);
1024 	if (err)
1025 		return err;
1026 
1027 	return got_privsep_recv_tree(tree, pack->privsep_child->ibuf);
1028 }
1029 
1030 static const struct got_error *
1031 read_packed_tree_privsep(struct got_tree_object **tree,
1032     struct got_pack *pack, struct got_packidx *packidx, int idx,
1033     struct got_object_id *id)
1034 {
1035 	const struct got_error *err = NULL;
1036 
1037 	if (pack->privsep_child)
1038 		return request_packed_tree(tree, pack, idx, id);
1039 
1040 	err = start_pack_privsep_child(pack, packidx);
1041 	if (err)
1042 		return err;
1043 
1044 	return request_packed_tree(tree, pack, idx, id);
1045 }
1046 
1047 static const struct got_error *
1048 request_tree(struct got_tree_object **tree, struct got_repository *repo,
1049     int fd, struct got_object_id *id)
1050 {
1051 	const struct got_error *err = NULL;
1052 	struct imsgbuf *ibuf;
1053 
1054 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].ibuf;
1055 
1056 	err = got_privsep_send_tree_req(ibuf, fd, id, -1);
1057 	if (err)
1058 		return err;
1059 
1060 	return got_privsep_recv_tree(tree, ibuf);
1061 }
1062 
1063 const struct got_error *
1064 read_tree_privsep(struct got_tree_object **tree, int obj_fd,
1065     struct got_object_id *id, struct got_repository *repo)
1066 {
1067 	const struct got_error *err;
1068 	int imsg_fds[2];
1069 	pid_t pid;
1070 	struct imsgbuf *ibuf;
1071 
1072 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].imsg_fd != -1)
1073 		return request_tree(tree, repo, obj_fd, id);
1074 
1075 	ibuf = calloc(1, sizeof(*ibuf));
1076 	if (ibuf == NULL)
1077 		return got_error_from_errno("calloc");
1078 
1079 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1080 		err = got_error_from_errno("socketpair");
1081 		free(ibuf);
1082 		return err;
1083 	}
1084 
1085 	pid = fork();
1086 	if (pid == -1) {
1087 		err = got_error_from_errno("fork");
1088 		free(ibuf);
1089 		return err;
1090 	}
1091 	else if (pid == 0) {
1092 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_TREE,
1093 		    repo->path);
1094 		/* not reached */
1095 	}
1096 
1097 	if (close(imsg_fds[1]) == -1) {
1098 		err = got_error_from_errno("close");
1099 		free(ibuf);
1100 		return err;
1101 	}
1102 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].imsg_fd =
1103 	    imsg_fds[0];
1104 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].pid = pid;
1105 	imsg_init(ibuf, imsg_fds[0]);
1106 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TREE].ibuf = ibuf;
1107 
1108 
1109 	return request_tree(tree, repo, obj_fd, id);
1110 }
1111 
1112 static const struct got_error *
1113 open_tree(struct got_tree_object **tree, struct got_repository *repo,
1114     struct got_object_id *id, int check_cache)
1115 {
1116 	const struct got_error *err = NULL;
1117 	struct got_packidx *packidx = NULL;
1118 	int idx;
1119 	char *path_packfile = NULL;
1120 
1121 	if (check_cache) {
1122 		*tree = got_repo_get_cached_tree(repo, id);
1123 		if (*tree != NULL) {
1124 			(*tree)->refcnt++;
1125 			return NULL;
1126 		}
1127 	} else
1128 		*tree = NULL;
1129 
1130 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1131 	if (err == NULL) {
1132 		struct got_pack *pack = NULL;
1133 
1134 		err = got_packidx_get_packfile_path(&path_packfile,
1135 		    packidx->path_packidx);
1136 		if (err)
1137 			return err;
1138 
1139 		pack = got_repo_get_cached_pack(repo, path_packfile);
1140 		if (pack == NULL) {
1141 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1142 			    packidx);
1143 			if (err)
1144 				goto done;
1145 		}
1146 		err = read_packed_tree_privsep(tree, pack,
1147 		    packidx, idx, id);
1148 	} else if (err->code == GOT_ERR_NO_OBJ) {
1149 		int fd;
1150 
1151 		err = got_object_open_loose_fd(&fd, id, repo);
1152 		if (err)
1153 			return err;
1154 		err = read_tree_privsep(tree, fd, id, repo);
1155 	}
1156 
1157 	if (err == NULL) {
1158 		(*tree)->refcnt++;
1159 		err = got_repo_cache_tree(repo, id, *tree);
1160 	}
1161 done:
1162 	free(path_packfile);
1163 	return err;
1164 }
1165 
1166 const struct got_error *
1167 got_object_open_as_tree(struct got_tree_object **tree,
1168     struct got_repository *repo, struct got_object_id *id)
1169 {
1170 	*tree = got_repo_get_cached_tree(repo, id);
1171 	if (*tree != NULL) {
1172 		(*tree)->refcnt++;
1173 		return NULL;
1174 	}
1175 
1176 	return open_tree(tree, repo, id, 0);
1177 }
1178 
1179 const struct got_error *
1180 got_object_tree_open(struct got_tree_object **tree,
1181     struct got_repository *repo, struct got_object *obj)
1182 {
1183 	return open_tree(tree, repo, got_object_get_id(obj), 1);
1184 }
1185 
1186 int
1187 got_object_tree_get_nentries(struct got_tree_object *tree)
1188 {
1189 	return tree->nentries;
1190 }
1191 
1192 struct got_tree_entry *
1193 got_object_tree_get_first_entry(struct got_tree_object *tree)
1194 {
1195 	return got_object_tree_get_entry(tree, 0);
1196 }
1197 
1198 struct got_tree_entry *
1199 got_object_tree_get_last_entry(struct got_tree_object *tree)
1200 {
1201 	return got_object_tree_get_entry(tree, tree->nentries - 1);
1202 }
1203 
1204 struct got_tree_entry *
1205 got_object_tree_get_entry(struct got_tree_object *tree, int i)
1206 {
1207 	if (i < 0 || i >= tree->nentries)
1208 		return NULL;
1209 	return &tree->entries[i];
1210 }
1211 
1212 mode_t
1213 got_tree_entry_get_mode(struct got_tree_entry *te)
1214 {
1215 	return te->mode;
1216 }
1217 
1218 const char *
1219 got_tree_entry_get_name(struct got_tree_entry *te)
1220 {
1221 	return &te->name[0];
1222 }
1223 
1224 struct got_object_id *
1225 got_tree_entry_get_id(struct got_tree_entry *te)
1226 {
1227 	return &te->id;
1228 }
1229 
1230 const struct got_error *
1231 got_object_blob_read_to_str(char **s, struct got_blob_object *blob)
1232 {
1233 	const struct got_error *err = NULL;
1234 	size_t len, totlen, hdrlen, offset;
1235 
1236 	*s = NULL;
1237 
1238 	hdrlen = got_object_blob_get_hdrlen(blob);
1239 	totlen = 0;
1240 	offset = 0;
1241 	do {
1242 		char *p;
1243 
1244 		err = got_object_blob_read_block(&len, blob);
1245 		if (err)
1246 			return err;
1247 
1248 		if (len == 0)
1249 			break;
1250 
1251 		totlen += len - hdrlen;
1252 		p = realloc(*s, totlen + 1);
1253 		if (p == NULL) {
1254 			err = got_error_from_errno("realloc");
1255 			free(*s);
1256 			*s = NULL;
1257 			return err;
1258 		}
1259 		*s = p;
1260 		/* Skip blob object header first time around. */
1261 		memcpy(*s + offset,
1262 		    got_object_blob_get_read_buf(blob) + hdrlen, len - hdrlen);
1263 		hdrlen = 0;
1264 		offset = totlen;
1265 	} while (len > 0);
1266 
1267 	(*s)[totlen] = '\0';
1268 	return NULL;
1269 }
1270 
1271 const struct got_error *
1272 got_tree_entry_get_symlink_target(char **link_target, struct got_tree_entry *te,
1273     struct got_repository *repo)
1274 {
1275 	const struct got_error *err = NULL;
1276 	struct got_blob_object *blob = NULL;
1277 
1278 	*link_target = NULL;
1279 
1280 	if (!got_object_tree_entry_is_symlink(te))
1281 		return got_error(GOT_ERR_TREE_ENTRY_TYPE);
1282 
1283 	err = got_object_open_as_blob(&blob, repo,
1284 	    got_tree_entry_get_id(te), PATH_MAX);
1285 	if (err)
1286 		return err;
1287 
1288 	err = got_object_blob_read_to_str(link_target, blob);
1289 	got_object_blob_close(blob);
1290 	if (err) {
1291 		free(*link_target);
1292 		*link_target = NULL;
1293 	}
1294 	return err;
1295 }
1296 
1297 int
1298 got_tree_entry_get_index(struct got_tree_entry *te)
1299 {
1300 	return te->idx;
1301 }
1302 
1303 struct got_tree_entry *
1304 got_tree_entry_get_next(struct got_tree_object *tree,
1305     struct got_tree_entry *te)
1306 {
1307 	return got_object_tree_get_entry(tree, te->idx + 1);
1308 }
1309 
1310 struct got_tree_entry *
1311 got_tree_entry_get_prev(struct got_tree_object *tree,
1312     struct got_tree_entry *te)
1313 {
1314 	return got_object_tree_get_entry(tree, te->idx - 1);
1315 }
1316 
1317 static const struct got_error *
1318 request_packed_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd,
1319     struct got_pack *pack, struct got_packidx *packidx, int idx,
1320     struct got_object_id *id)
1321 {
1322 	const struct got_error *err = NULL;
1323 	struct imsgbuf *ibuf = pack->privsep_child->ibuf;
1324 	int outfd_child;
1325 
1326 	 err = pack_child_send_tempfiles(ibuf, pack);
1327 	 if (err)
1328 		return err;
1329 
1330 	outfd_child = dup(outfd);
1331 	if (outfd_child == -1)
1332 		return got_error_from_errno("dup");
1333 
1334 	err = got_privsep_send_blob_req(pack->privsep_child->ibuf, -1, id, idx);
1335 	if (err)
1336 		return err;
1337 
1338 	err = got_privsep_send_blob_outfd(pack->privsep_child->ibuf,
1339 	    outfd_child);
1340 	if (err) {
1341 		return err;
1342 	}
1343 
1344 	err = got_privsep_recv_blob(outbuf, size, hdrlen,
1345 	    pack->privsep_child->ibuf);
1346 	if (err)
1347 		return err;
1348 
1349 	if (lseek(outfd, SEEK_SET, 0) == -1)
1350 		err = got_error_from_errno("lseek");
1351 
1352 	return err;
1353 }
1354 
1355 static const struct got_error *
1356 read_packed_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen,
1357     int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
1358     struct got_object_id *id)
1359 {
1360 	const struct got_error *err = NULL;
1361 
1362 	if (pack->privsep_child == NULL) {
1363 		err = start_pack_privsep_child(pack, packidx);
1364 		if (err)
1365 			return err;
1366 	}
1367 
1368 	return request_packed_blob(outbuf, size, hdrlen, outfd, pack, packidx,
1369 	    idx, id);
1370 }
1371 
1372 static const struct got_error *
1373 request_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd,
1374     int infd, struct got_object_id *id, struct imsgbuf *ibuf)
1375 {
1376 	const struct got_error *err = NULL;
1377 	int outfd_child;
1378 
1379 	outfd_child = dup(outfd);
1380 	if (outfd_child == -1)
1381 		return got_error_from_errno("dup");
1382 
1383 	err = got_privsep_send_blob_req(ibuf, infd, id, -1);
1384 	if (err)
1385 		return err;
1386 
1387 	err = got_privsep_send_blob_outfd(ibuf, outfd_child);
1388 	if (err)
1389 		return err;
1390 
1391 	err = got_privsep_recv_blob(outbuf, size, hdrlen, ibuf);
1392 	if (err)
1393 		return err;
1394 
1395 	if (lseek(outfd, SEEK_SET, 0) == -1)
1396 		return got_error_from_errno("lseek");
1397 
1398 	return err;
1399 }
1400 
1401 static const struct got_error *
1402 read_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen,
1403     int outfd, int infd, struct got_object_id *id, struct got_repository *repo)
1404 {
1405 	const struct got_error *err;
1406 	int imsg_fds[2];
1407 	pid_t pid;
1408 	struct imsgbuf *ibuf;
1409 
1410 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) {
1411 		ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf;
1412 		return request_blob(outbuf, size, hdrlen, outfd, infd, id,
1413 		    ibuf);
1414 	}
1415 
1416 	ibuf = calloc(1, sizeof(*ibuf));
1417 	if (ibuf == NULL)
1418 		return got_error_from_errno("calloc");
1419 
1420 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1421 		err = got_error_from_errno("socketpair");
1422 		free(ibuf);
1423 		return err;
1424 	}
1425 
1426 	pid = fork();
1427 	if (pid == -1) {
1428 		err = got_error_from_errno("fork");
1429 		free(ibuf);
1430 		return err;
1431 	}
1432 	else if (pid == 0) {
1433 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_BLOB,
1434 		    repo->path);
1435 		/* not reached */
1436 	}
1437 
1438 	if (close(imsg_fds[1]) == -1) {
1439 		err = got_error_from_errno("close");
1440 		free(ibuf);
1441 		return err;
1442 	}
1443 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd =
1444 	    imsg_fds[0];
1445 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].pid = pid;
1446 	imsg_init(ibuf, imsg_fds[0]);
1447 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf;
1448 
1449 	return request_blob(outbuf, size, hdrlen, outfd, infd, id, ibuf);
1450 }
1451 
1452 static const struct got_error *
1453 open_blob(struct got_blob_object **blob, struct got_repository *repo,
1454     struct got_object_id *id, size_t blocksize)
1455 {
1456 	const struct got_error *err = NULL;
1457 	struct got_packidx *packidx = NULL;
1458 	int idx;
1459 	char *path_packfile = NULL;
1460 	uint8_t *outbuf;
1461 	int outfd;
1462 	size_t size, hdrlen;
1463 	struct stat sb;
1464 
1465 	*blob = calloc(1, sizeof(**blob));
1466 	if (*blob == NULL)
1467 		return got_error_from_errno("calloc");
1468 
1469 	outfd = got_opentempfd();
1470 	if (outfd == -1)
1471 		return got_error_from_errno("got_opentempfd");
1472 
1473 	(*blob)->read_buf = malloc(blocksize);
1474 	if ((*blob)->read_buf == NULL) {
1475 		err = got_error_from_errno("malloc");
1476 		goto done;
1477 	}
1478 
1479 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1480 	if (err == NULL) {
1481 		struct got_pack *pack = NULL;
1482 
1483 		err = got_packidx_get_packfile_path(&path_packfile,
1484 		    packidx->path_packidx);
1485 		if (err)
1486 			goto done;
1487 
1488 		pack = got_repo_get_cached_pack(repo, path_packfile);
1489 		if (pack == NULL) {
1490 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1491 			    packidx);
1492 			if (err)
1493 				goto done;
1494 		}
1495 		err = read_packed_blob_privsep(&outbuf, &size, &hdrlen, outfd,
1496 		    pack, packidx, idx, id);
1497 	} else if (err->code == GOT_ERR_NO_OBJ) {
1498 		int infd;
1499 
1500 		err = got_object_open_loose_fd(&infd, id, repo);
1501 		if (err)
1502 			goto done;
1503 		err = read_blob_privsep(&outbuf, &size, &hdrlen, outfd, infd,
1504 		    id, repo);
1505 	}
1506 	if (err)
1507 		goto done;
1508 
1509 	if (hdrlen > size) {
1510 		err = got_error(GOT_ERR_BAD_OBJ_HDR);
1511 		goto done;
1512 	}
1513 
1514 	if (outbuf) {
1515 		if (close(outfd) == -1 && err == NULL)
1516 			err = got_error_from_errno("close");
1517 		outfd = -1;
1518 		(*blob)->f = fmemopen(outbuf, size, "rb");
1519 		if ((*blob)->f == NULL) {
1520 			err = got_error_from_errno("fmemopen");
1521 			free(outbuf);
1522 			goto done;
1523 		}
1524 		(*blob)->data = outbuf;
1525 	} else {
1526 		if (fstat(outfd, &sb) == -1) {
1527 			err = got_error_from_errno("fstat");
1528 			goto done;
1529 		}
1530 
1531 		if (sb.st_size != size) {
1532 			err = got_error(GOT_ERR_PRIVSEP_LEN);
1533 			goto done;
1534 		}
1535 
1536 		(*blob)->f = fdopen(outfd, "rb");
1537 		if ((*blob)->f == NULL) {
1538 			err = got_error_from_errno("fdopen");
1539 			close(outfd);
1540 			outfd = -1;
1541 			goto done;
1542 		}
1543 	}
1544 
1545 	(*blob)->hdrlen = hdrlen;
1546 	(*blob)->blocksize = blocksize;
1547 	memcpy(&(*blob)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH);
1548 
1549 done:
1550 	free(path_packfile);
1551 	if (err) {
1552 		if (*blob) {
1553 			got_object_blob_close(*blob);
1554 			*blob = NULL;
1555 		} else if (outfd != -1)
1556 			close(outfd);
1557 	}
1558 	return err;
1559 }
1560 
1561 const struct got_error *
1562 got_object_open_as_blob(struct got_blob_object **blob,
1563     struct got_repository *repo, struct got_object_id *id,
1564     size_t blocksize)
1565 {
1566 	return open_blob(blob, repo, id, blocksize);
1567 }
1568 
1569 const struct got_error *
1570 got_object_blob_open(struct got_blob_object **blob,
1571     struct got_repository *repo, struct got_object *obj, size_t blocksize)
1572 {
1573 	return open_blob(blob, repo, got_object_get_id(obj), blocksize);
1574 }
1575 
1576 const struct got_error *
1577 got_object_blob_close(struct got_blob_object *blob)
1578 {
1579 	const struct got_error *err = NULL;
1580 	free(blob->read_buf);
1581 	if (blob->f && fclose(blob->f) == EOF)
1582 		err = got_error_from_errno("fclose");
1583 	free(blob->data);
1584 	free(blob);
1585 	return err;
1586 }
1587 
1588 void
1589 got_object_blob_rewind(struct got_blob_object *blob)
1590 {
1591 	if (blob->f)
1592 		rewind(blob->f);
1593 }
1594 
1595 char *
1596 got_object_blob_id_str(struct got_blob_object *blob, char *buf, size_t size)
1597 {
1598 	return got_sha1_digest_to_str(blob->id.sha1, buf, size);
1599 }
1600 
1601 size_t
1602 got_object_blob_get_hdrlen(struct got_blob_object *blob)
1603 {
1604 	return blob->hdrlen;
1605 }
1606 
1607 const uint8_t *
1608 got_object_blob_get_read_buf(struct got_blob_object *blob)
1609 {
1610 	return blob->read_buf;
1611 }
1612 
1613 const struct got_error *
1614 got_object_blob_read_block(size_t *outlenp, struct got_blob_object *blob)
1615 {
1616 	size_t n;
1617 
1618 	n = fread(blob->read_buf, 1, blob->blocksize, blob->f);
1619 	if (n == 0 && ferror(blob->f))
1620 		return got_ferror(blob->f, GOT_ERR_IO);
1621 	*outlenp = n;
1622 	return NULL;
1623 }
1624 
1625 const struct got_error *
1626 got_object_blob_dump_to_file(off_t *filesize, int *nlines,
1627     off_t **line_offsets, FILE *outfile, struct got_blob_object *blob)
1628 {
1629 	const struct got_error *err = NULL;
1630 	size_t n, len, hdrlen;
1631 	const uint8_t *buf;
1632 	int i;
1633 	const int alloc_chunksz = 512;
1634 	size_t nalloc = 0;
1635 	off_t off = 0, total_len = 0;
1636 
1637 	if (line_offsets)
1638 		*line_offsets = NULL;
1639 	if (filesize)
1640 		*filesize = 0;
1641 	if (nlines)
1642 		*nlines = 0;
1643 
1644 	hdrlen = got_object_blob_get_hdrlen(blob);
1645 	do {
1646 		err = got_object_blob_read_block(&len, blob);
1647 		if (err)
1648 			return err;
1649 		if (len == 0)
1650 			break;
1651 		buf = got_object_blob_get_read_buf(blob);
1652 		i = hdrlen;
1653 		if (nlines) {
1654 			if (line_offsets && *line_offsets == NULL) {
1655 				/* Have some data but perhaps no '\n'. */
1656 				*nlines = 1;
1657 				nalloc = alloc_chunksz;
1658 				*line_offsets = calloc(nalloc,
1659 				    sizeof(**line_offsets));
1660 				if (*line_offsets == NULL)
1661 					return got_error_from_errno("calloc");
1662 
1663 				/* Skip forward over end of first line. */
1664 				while (i < len) {
1665 					if (buf[i] == '\n')
1666 						break;
1667 					i++;
1668 				}
1669 			}
1670 			/* Scan '\n' offsets in remaining chunk of data. */
1671 			while (i < len) {
1672 				if (buf[i] != '\n') {
1673 					i++;
1674 					continue;
1675 				}
1676 				(*nlines)++;
1677 				if (line_offsets && nalloc < *nlines) {
1678 					size_t n = *nlines + alloc_chunksz;
1679 					off_t *o = recallocarray(*line_offsets,
1680 					    nalloc, n, sizeof(**line_offsets));
1681 					if (o == NULL) {
1682 						free(*line_offsets);
1683 						*line_offsets = NULL;
1684 						return got_error_from_errno(
1685 						    "recallocarray");
1686 					}
1687 					*line_offsets = o;
1688 					nalloc = n;
1689 				}
1690 				if (line_offsets) {
1691 					off = total_len + i - hdrlen + 1;
1692 					(*line_offsets)[*nlines - 1] = off;
1693 				}
1694 				i++;
1695 			}
1696 		}
1697 		/* Skip blob object header first time around. */
1698 		n = fwrite(buf + hdrlen, 1, len - hdrlen, outfile);
1699 		if (n != len - hdrlen)
1700 			return got_ferror(outfile, GOT_ERR_IO);
1701 		total_len += len - hdrlen;
1702 		hdrlen = 0;
1703 	} while (len != 0);
1704 
1705 	if (fflush(outfile) != 0)
1706 		return got_error_from_errno("fflush");
1707 	rewind(outfile);
1708 
1709 	if (filesize)
1710 		*filesize = total_len;
1711 
1712 	return NULL;
1713 }
1714 
1715 static const struct got_error *
1716 request_packed_tag(struct got_tag_object **tag, struct got_pack *pack,
1717     int pack_idx, struct got_object_id *id)
1718 {
1719 	const struct got_error *err = NULL;
1720 
1721 	err = got_privsep_send_tag_req(pack->privsep_child->ibuf, -1, id,
1722 	    pack_idx);
1723 	if (err)
1724 		return err;
1725 
1726 	return got_privsep_recv_tag(tag, pack->privsep_child->ibuf);
1727 }
1728 
1729 static const struct got_error *
1730 read_packed_tag_privsep(struct got_tag_object **tag,
1731     struct got_pack *pack, struct got_packidx *packidx, int idx,
1732     struct got_object_id *id)
1733 {
1734 	const struct got_error *err = NULL;
1735 
1736 	if (pack->privsep_child)
1737 		return request_packed_tag(tag, pack, idx, id);
1738 
1739 	err = start_pack_privsep_child(pack, packidx);
1740 	if (err)
1741 		return err;
1742 
1743 	return request_packed_tag(tag, pack, idx, id);
1744 }
1745 
1746 static const struct got_error *
1747 request_tag(struct got_tag_object **tag, struct got_repository *repo,
1748     int fd, struct got_object_id *id)
1749 {
1750 	const struct got_error *err = NULL;
1751 	struct imsgbuf *ibuf;
1752 
1753 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].ibuf;
1754 
1755 	err = got_privsep_send_tag_req(ibuf, fd, id, -1);
1756 	if (err)
1757 		return err;
1758 
1759 	return got_privsep_recv_tag(tag, ibuf);
1760 }
1761 
1762 static const struct got_error *
1763 read_tag_privsep(struct got_tag_object **tag, int obj_fd,
1764     struct got_object_id *id, struct got_repository *repo)
1765 {
1766 	const struct got_error *err;
1767 	int imsg_fds[2];
1768 	pid_t pid;
1769 	struct imsgbuf *ibuf;
1770 
1771 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].imsg_fd != -1)
1772 		return request_tag(tag, repo, obj_fd, id);
1773 
1774 	ibuf = calloc(1, sizeof(*ibuf));
1775 	if (ibuf == NULL)
1776 		return got_error_from_errno("calloc");
1777 
1778 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) {
1779 		err = got_error_from_errno("socketpair");
1780 		free(ibuf);
1781 		return err;
1782 	}
1783 
1784 	pid = fork();
1785 	if (pid == -1) {
1786 		err = got_error_from_errno("fork");
1787 		free(ibuf);
1788 		return err;
1789 	}
1790 	else if (pid == 0) {
1791 		got_privsep_exec_child(imsg_fds, GOT_PATH_PROG_READ_TAG,
1792 		    repo->path);
1793 		/* not reached */
1794 	}
1795 
1796 	if (close(imsg_fds[1]) == -1) {
1797 		err = got_error_from_errno("close");
1798 		free(ibuf);
1799 		return err;
1800 	}
1801 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].imsg_fd =
1802 	    imsg_fds[0];
1803 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].pid = pid;
1804 	imsg_init(ibuf, imsg_fds[0]);
1805 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_TAG].ibuf = ibuf;
1806 
1807 	return request_tag(tag, repo, obj_fd, id);
1808 }
1809 
1810 static const struct got_error *
1811 open_tag(struct got_tag_object **tag, struct got_repository *repo,
1812     struct got_object_id *id, int check_cache)
1813 {
1814 	const struct got_error *err = NULL;
1815 	struct got_packidx *packidx = NULL;
1816 	int idx;
1817 	char *path_packfile = NULL;
1818 	struct got_object *obj = NULL;
1819 	int obj_type = GOT_OBJ_TYPE_ANY;
1820 
1821 	if (check_cache) {
1822 		*tag = got_repo_get_cached_tag(repo, id);
1823 		if (*tag != NULL) {
1824 			(*tag)->refcnt++;
1825 			return NULL;
1826 		}
1827 	} else
1828 		*tag = NULL;
1829 
1830 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
1831 	if (err == NULL) {
1832 		struct got_pack *pack = NULL;
1833 
1834 		err = got_packidx_get_packfile_path(&path_packfile,
1835 		    packidx->path_packidx);
1836 		if (err)
1837 			return err;
1838 
1839 		pack = got_repo_get_cached_pack(repo, path_packfile);
1840 		if (pack == NULL) {
1841 			err = got_repo_cache_pack(&pack, repo, path_packfile,
1842 			    packidx);
1843 			if (err)
1844 				goto done;
1845 		}
1846 
1847 		/* Beware of "lightweight" tags: Check object type first. */
1848 		err = read_packed_object_privsep(&obj, repo, pack, packidx,
1849 		    idx, id);
1850 		if (err)
1851 			goto done;
1852 		obj_type = obj->type;
1853 		got_object_close(obj);
1854 		if (obj_type != GOT_OBJ_TYPE_TAG) {
1855 			err = got_error(GOT_ERR_OBJ_TYPE);
1856 			goto done;
1857 		}
1858 		err = read_packed_tag_privsep(tag, pack, packidx, idx, id);
1859 	} else if (err->code == GOT_ERR_NO_OBJ) {
1860 		int fd;
1861 
1862 		err = got_object_open_loose_fd(&fd, id, repo);
1863 		if (err)
1864 			return err;
1865 		err = got_object_read_header_privsep(&obj, id, repo, fd);
1866 		if (err)
1867 			return err;
1868 		obj_type = obj->type;
1869 		got_object_close(obj);
1870 		if (obj_type != GOT_OBJ_TYPE_TAG)
1871 			return got_error(GOT_ERR_OBJ_TYPE);
1872 
1873 		err = got_object_open_loose_fd(&fd, id, repo);
1874 		if (err)
1875 			return err;
1876 		err = read_tag_privsep(tag, fd, id, repo);
1877 	}
1878 
1879 	if (err == NULL) {
1880 		(*tag)->refcnt++;
1881 		err = got_repo_cache_tag(repo, id, *tag);
1882 	}
1883 done:
1884 	free(path_packfile);
1885 	return err;
1886 }
1887 
1888 const struct got_error *
1889 got_object_open_as_tag(struct got_tag_object **tag,
1890     struct got_repository *repo, struct got_object_id *id)
1891 {
1892 	*tag = got_repo_get_cached_tag(repo, id);
1893 	if (*tag != NULL) {
1894 		(*tag)->refcnt++;
1895 		return NULL;
1896 	}
1897 
1898 	return open_tag(tag, repo, id, 0);
1899 }
1900 
1901 const struct got_error *
1902 got_object_tag_open(struct got_tag_object **tag,
1903     struct got_repository *repo, struct got_object *obj)
1904 {
1905 	return open_tag(tag, repo, got_object_get_id(obj), 1);
1906 }
1907 
1908 const char *
1909 got_object_tag_get_name(struct got_tag_object *tag)
1910 {
1911 	return tag->tag;
1912 }
1913 
1914 int
1915 got_object_tag_get_object_type(struct got_tag_object *tag)
1916 {
1917 	return tag->obj_type;
1918 }
1919 
1920 struct got_object_id *
1921 got_object_tag_get_object_id(struct got_tag_object *tag)
1922 {
1923 	return &tag->id;
1924 }
1925 
1926 time_t
1927 got_object_tag_get_tagger_time(struct got_tag_object *tag)
1928 {
1929 	return tag->tagger_time;
1930 }
1931 
1932 time_t
1933 got_object_tag_get_tagger_gmtoff(struct got_tag_object *tag)
1934 {
1935 	return tag->tagger_gmtoff;
1936 }
1937 
1938 const char *
1939 got_object_tag_get_tagger(struct got_tag_object *tag)
1940 {
1941 	return tag->tagger;
1942 }
1943 
1944 const char *
1945 got_object_tag_get_message(struct got_tag_object *tag)
1946 {
1947 	return tag->tagmsg;
1948 }
1949 
1950 static struct got_tree_entry *
1951 find_entry_by_name(struct got_tree_object *tree, const char *name, size_t len)
1952 {
1953 	int i;
1954 
1955 	/* Note that tree entries are sorted in strncmp() order. */
1956 	for (i = 0; i < tree->nentries; i++) {
1957 		struct got_tree_entry *te = &tree->entries[i];
1958 		int cmp = strncmp(te->name, name, len);
1959 		if (cmp < 0)
1960 			continue;
1961 		if (cmp > 0)
1962 			break;
1963 		if (te->name[len] == '\0')
1964 			return te;
1965 	}
1966 	return NULL;
1967 }
1968 
1969 struct got_tree_entry *
1970 got_object_tree_find_entry(struct got_tree_object *tree, const char *name)
1971 {
1972 	return find_entry_by_name(tree, name, strlen(name));
1973 }
1974 
1975 const struct got_error *
1976 got_object_tree_find_path(struct got_object_id **id, mode_t *mode,
1977     struct got_repository *repo, struct got_tree_object *tree,
1978     const char *path)
1979 {
1980 	const struct got_error *err = NULL;
1981 	struct got_tree_object *subtree = NULL;
1982 	struct got_tree_entry *te = NULL;
1983 	const char *seg, *s;
1984 	size_t seglen;
1985 
1986 	*id = NULL;
1987 
1988 	s = path;
1989 	while (s[0] == '/')
1990 		s++;
1991 	seg = s;
1992 	seglen = 0;
1993 	subtree = tree;
1994 	while (*s) {
1995 		struct got_tree_object *next_tree;
1996 
1997 		if (*s != '/') {
1998 			s++;
1999 			seglen++;
2000 			if (*s)
2001 				continue;
2002 		}
2003 
2004 		te = find_entry_by_name(subtree, seg, seglen);
2005 		if (te == NULL) {
2006 			err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
2007 			goto done;
2008 		}
2009 
2010 		if (*s == '\0')
2011 			break;
2012 
2013 		seg = s + 1;
2014 		seglen = 0;
2015 		s++;
2016 		if (*s) {
2017 			err = got_object_open_as_tree(&next_tree, repo,
2018 			    &te->id);
2019 			te = NULL;
2020 			if (err)
2021 				goto done;
2022 			if (subtree != tree)
2023 				got_object_tree_close(subtree);
2024 			subtree = next_tree;
2025 		}
2026 	}
2027 
2028 	if (te) {
2029 		*id = got_object_id_dup(&te->id);
2030 		if (*id == NULL)
2031 			return got_error_from_errno("got_object_id_dup");
2032 		if (mode)
2033 			*mode = te->mode;
2034 	} else
2035 		err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
2036 done:
2037 	if (subtree && subtree != tree)
2038 		got_object_tree_close(subtree);
2039 	return err;
2040 }
2041 const struct got_error *
2042 got_object_id_by_path(struct got_object_id **id, struct got_repository *repo,
2043     struct got_commit_object *commit, const char *path)
2044 {
2045 	const struct got_error *err = NULL;
2046 	struct got_tree_object *tree = NULL;
2047 
2048 	*id = NULL;
2049 
2050 	/* Handle opening of root of commit's tree. */
2051 	if (got_path_is_root_dir(path)) {
2052 		*id = got_object_id_dup(commit->tree_id);
2053 		if (*id == NULL)
2054 			err = got_error_from_errno("got_object_id_dup");
2055 	} else {
2056 		err = got_object_open_as_tree(&tree, repo, commit->tree_id);
2057 		if (err)
2058 			goto done;
2059 		err = got_object_tree_find_path(id, NULL, repo, tree, path);
2060 	}
2061 done:
2062 	if (tree)
2063 		got_object_tree_close(tree);
2064 	return err;
2065 }
2066 
2067 /*
2068  * Normalize file mode bits to avoid false positive tree entry differences
2069  * in case tree entries have unexpected mode bits set.
2070  */
2071 static mode_t
2072 normalize_mode_for_comparison(mode_t mode)
2073 {
2074 	/*
2075 	 * For directories, the only relevant bit is the IFDIR bit.
2076 	 * This allows us to detect paths changing from a directory
2077 	 * to a file and vice versa.
2078 	 */
2079 	if (S_ISDIR(mode))
2080 		return mode & S_IFDIR;
2081 
2082 	/*
2083 	 * For symlinks, the only relevant bit is the IFLNK bit.
2084 	 * This allows us to detect paths changing from a symlinks
2085 	 * to a file or directory and vice versa.
2086 	 */
2087 	if (S_ISLNK(mode))
2088 		return mode & S_IFLNK;
2089 
2090 	/* For files, the only change we care about is the executable bit. */
2091 	return mode & S_IXUSR;
2092 }
2093 
2094 const struct got_error *
2095 got_object_tree_path_changed(int *changed,
2096     struct got_tree_object *tree01, struct got_tree_object *tree02,
2097     const char *path, struct got_repository *repo)
2098 {
2099 	const struct got_error *err = NULL;
2100 	struct got_tree_object *tree1 = NULL, *tree2 = NULL;
2101 	struct got_tree_entry *te1 = NULL, *te2 = NULL;
2102 	const char *seg, *s;
2103 	size_t seglen;
2104 
2105 	*changed = 0;
2106 
2107 	/* We not do support comparing the root path. */
2108 	if (got_path_is_root_dir(path))
2109 		return got_error_path(path, GOT_ERR_BAD_PATH);
2110 
2111 	tree1 = tree01;
2112 	tree2 = tree02;
2113 	s = path;
2114 	while (*s == '/')
2115 		s++;
2116 	seg = s;
2117 	seglen = 0;
2118 	while (*s) {
2119 		struct got_tree_object *next_tree1, *next_tree2;
2120 		mode_t mode1, mode2;
2121 
2122 		if (*s != '/') {
2123 			s++;
2124 			seglen++;
2125 			if (*s)
2126 				continue;
2127 		}
2128 
2129 		te1 = find_entry_by_name(tree1, seg, seglen);
2130 		if (te1 == NULL) {
2131 			err = got_error(GOT_ERR_NO_OBJ);
2132 			goto done;
2133 		}
2134 
2135 		if (tree2)
2136 			te2 = find_entry_by_name(tree2, seg, seglen);
2137 
2138 		if (te2) {
2139 			mode1 = normalize_mode_for_comparison(te1->mode);
2140 			mode2 = normalize_mode_for_comparison(te2->mode);
2141 			if (mode1 != mode2) {
2142 				*changed = 1;
2143 				goto done;
2144 			}
2145 
2146 			if (got_object_id_cmp(&te1->id, &te2->id) == 0) {
2147 				*changed = 0;
2148 				goto done;
2149 			}
2150 		}
2151 
2152 		if (*s == '\0') { /* final path element */
2153 			*changed = 1;
2154 			goto done;
2155 		}
2156 
2157 		seg = s + 1;
2158 		s++;
2159 		seglen = 0;
2160 		if (*s) {
2161 			err = got_object_open_as_tree(&next_tree1, repo,
2162 			    &te1->id);
2163 			te1 = NULL;
2164 			if (err)
2165 				goto done;
2166 			if (tree1 != tree01)
2167 				got_object_tree_close(tree1);
2168 			tree1 = next_tree1;
2169 
2170 			if (te2) {
2171 				err = got_object_open_as_tree(&next_tree2, repo,
2172 				    &te2->id);
2173 				te2 = NULL;
2174 				if (err)
2175 					goto done;
2176 				if (tree2 != tree02)
2177 					got_object_tree_close(tree2);
2178 				tree2 = next_tree2;
2179 			} else if (tree2) {
2180 				if (tree2 != tree02)
2181 					got_object_tree_close(tree2);
2182 				tree2 = NULL;
2183 			}
2184 		}
2185 	}
2186 done:
2187 	if (tree1 && tree1 != tree01)
2188 		got_object_tree_close(tree1);
2189 	if (tree2 && tree2 != tree02)
2190 		got_object_tree_close(tree2);
2191 	return err;
2192 }
2193 
2194 const struct got_error *
2195 got_object_tree_entry_dup(struct got_tree_entry **new_te,
2196     struct got_tree_entry *te)
2197 {
2198 	const struct got_error *err = NULL;
2199 
2200 	*new_te = calloc(1, sizeof(**new_te));
2201 	if (*new_te == NULL)
2202 		return got_error_from_errno("calloc");
2203 
2204 	(*new_te)->mode = te->mode;
2205 	memcpy((*new_te)->name, te->name, sizeof((*new_te)->name));
2206 	memcpy(&(*new_te)->id, &te->id, sizeof((*new_te)->id));
2207 	return err;
2208 }
2209 
2210 int
2211 got_object_tree_entry_is_submodule(struct got_tree_entry *te)
2212 {
2213 	return (te->mode & S_IFMT) == (S_IFDIR | S_IFLNK);
2214 }
2215 
2216 int
2217 got_object_tree_entry_is_symlink(struct got_tree_entry *te)
2218 {
2219 	/* S_IFDIR check avoids confusing symlinks with submodules. */
2220 	return ((te->mode & (S_IFDIR | S_IFLNK)) == S_IFLNK);
2221 }
2222 
2223 static const struct got_error *
2224 resolve_symlink(char **link_target, const char *path,
2225     struct got_commit_object *commit, struct got_repository *repo)
2226 {
2227 	const struct got_error *err = NULL;
2228 	char buf[PATH_MAX];
2229 	char *name, *parent_path = NULL;
2230 	struct got_object_id *tree_obj_id = NULL;
2231 	struct got_tree_object *tree = NULL;
2232 	struct got_tree_entry *te = NULL;
2233 
2234 	*link_target = NULL;
2235 
2236 	if (strlcpy(buf, path, sizeof(buf)) >= sizeof(buf))
2237 		return got_error(GOT_ERR_NO_SPACE);
2238 
2239 	name = basename(buf);
2240 	if (name == NULL)
2241 		return got_error_from_errno2("basename", path);
2242 
2243 	err = got_path_dirname(&parent_path, path);
2244 	if (err)
2245 		return err;
2246 
2247 	err = got_object_id_by_path(&tree_obj_id, repo, commit,
2248 	    parent_path);
2249 	if (err) {
2250 		if (err->code == GOT_ERR_NO_TREE_ENTRY) {
2251 			/* Display the complete path in error message. */
2252 			err = got_error_path(path, err->code);
2253 		}
2254 		goto done;
2255 	}
2256 
2257 	err = got_object_open_as_tree(&tree, repo, tree_obj_id);
2258 	if (err)
2259 		goto done;
2260 
2261 	te = got_object_tree_find_entry(tree, name);
2262 	if (te == NULL) {
2263 		err = got_error_path(path, GOT_ERR_NO_TREE_ENTRY);
2264 		goto done;
2265 	}
2266 
2267 	if (got_object_tree_entry_is_symlink(te)) {
2268 		err = got_tree_entry_get_symlink_target(link_target, te, repo);
2269 		if (err)
2270 			goto done;
2271 		if (!got_path_is_absolute(*link_target)) {
2272 			char *abspath;
2273 			if (asprintf(&abspath, "%s/%s", parent_path,
2274 			    *link_target) == -1) {
2275 				err = got_error_from_errno("asprintf");
2276 				goto done;
2277 			}
2278 			free(*link_target);
2279 			*link_target = malloc(PATH_MAX);
2280 			if (*link_target == NULL) {
2281 				err = got_error_from_errno("malloc");
2282 				goto done;
2283 			}
2284 			err = got_canonpath(abspath, *link_target, PATH_MAX);
2285 			free(abspath);
2286 			if (err)
2287 				goto done;
2288 		}
2289 	}
2290 done:
2291 	free(tree_obj_id);
2292 	if (tree)
2293 		got_object_tree_close(tree);
2294 	if (err) {
2295 		free(*link_target);
2296 		*link_target = NULL;
2297 	}
2298 	return err;
2299 }
2300 
2301 const struct got_error *
2302 got_object_resolve_symlinks(char **link_target, const char *path,
2303     struct got_commit_object *commit, struct got_repository *repo)
2304 {
2305 	const struct got_error *err = NULL;
2306 	char *next_target = NULL;
2307 	int max_recursion = 40; /* matches Git */
2308 
2309 	*link_target = NULL;
2310 
2311 	do {
2312 		err = resolve_symlink(&next_target,
2313 		    *link_target ? *link_target : path, commit, repo);
2314 		if (err)
2315 			break;
2316 		if (next_target) {
2317 			free(*link_target);
2318 			if (--max_recursion == 0) {
2319 				err = got_error_path(path, GOT_ERR_RECURSION);
2320 				*link_target = NULL;
2321 				break;
2322 			}
2323 			*link_target = next_target;
2324 		}
2325 	} while (next_target);
2326 
2327 	return err;
2328 }
2329 
2330 const struct got_error *
2331 got_traverse_packed_commits(struct got_object_id_queue *traversed_commits,
2332     struct got_object_id *commit_id, const char *path,
2333     struct got_repository *repo)
2334 {
2335 	const struct got_error *err = NULL;
2336 	struct got_pack *pack = NULL;
2337 	struct got_packidx *packidx = NULL;
2338 	char *path_packfile = NULL;
2339 	struct got_commit_object *changed_commit = NULL;
2340 	struct got_object_id *changed_commit_id = NULL;
2341 	int idx;
2342 
2343 	err = got_repo_search_packidx(&packidx, &idx, repo, commit_id);
2344 	if (err) {
2345 		if (err->code != GOT_ERR_NO_OBJ)
2346 			return err;
2347 		return NULL;
2348 	}
2349 
2350 	err = got_packidx_get_packfile_path(&path_packfile,
2351 	    packidx->path_packidx);
2352 	if (err)
2353 		return err;
2354 
2355 	pack = got_repo_get_cached_pack(repo, path_packfile);
2356 	if (pack == NULL) {
2357 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
2358 		if (err)
2359 			goto done;
2360 	}
2361 
2362 	if (pack->privsep_child == NULL) {
2363 		err = start_pack_privsep_child(pack, packidx);
2364 		if (err)
2365 			goto done;
2366 	}
2367 
2368 	err = got_privsep_send_commit_traversal_request(
2369 	    pack->privsep_child->ibuf, commit_id, idx, path);
2370 	if (err)
2371 		goto done;
2372 
2373 	err = got_privsep_recv_traversed_commits(&changed_commit,
2374 	    &changed_commit_id, traversed_commits, pack->privsep_child->ibuf);
2375 	if (err)
2376 		goto done;
2377 
2378 	if (changed_commit) {
2379 		/*
2380 		 * Cache the commit in which the path was changed.
2381 		 * This commit might be opened again soon.
2382 		 */
2383 		changed_commit->refcnt++;
2384 		err = got_repo_cache_commit(repo, changed_commit_id,
2385 		    changed_commit);
2386 		got_object_commit_close(changed_commit);
2387 	}
2388 done:
2389 	free(path_packfile);
2390 	free(changed_commit_id);
2391 	return err;
2392 }