Got Mirror

Blob

Date:: Tue Jan 4 16:11:52 2022 UTC
Message:: avoid the creation of new temporary files whenever a packed object is read This speeds up the creation of pack files by about 30%.
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/mman.h>
22 
23 #include <limits.h>
24 #include <signal.h>
25 #include <stdint.h>
26 #include <imsg.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <sha1.h>
31 #include <unistd.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
37 
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_privsep.h"
44 #include "got_lib_pack.h"
45 
46 static volatile sig_atomic_t sigint_received;
47 
48 static void
49 catch_sigint(int signo)
50 {
51 	sigint_received = 1;
52 }
53 
54 static const struct got_error *
55 open_object(struct got_object **obj, struct got_pack *pack,
56     struct got_packidx *packidx, int idx, struct got_object_id *id,
57     struct got_object_cache *objcache)
58 {
59 	const struct got_error *err;
60 
61 	err = got_packfile_open_object(obj, pack, packidx, idx, id);
62 	if (err)
63 		return err;
64 	(*obj)->refcnt++;
65 
66 	err = got_object_cache_add(objcache, id, *obj);
67 	if (err) {
68 		if (err->code == GOT_ERR_OBJ_EXISTS ||
69 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
70 			err = NULL;
71 		return err;
72 	}
73 	(*obj)->refcnt++;
74 	return NULL;
75 }
76 
77 static const struct got_error *
78 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
79     struct got_packidx *packidx, struct got_object_cache *objcache)
80 {
81 	const struct got_error *err = NULL;
82 	struct got_imsg_packed_object iobj;
83 	struct got_object *obj;
84 	struct got_object_id id;
85 	size_t datalen;
86 
87 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
88 	if (datalen != sizeof(iobj))
89 		return got_error(GOT_ERR_PRIVSEP_LEN);
90 	memcpy(&iobj, imsg->data, sizeof(iobj));
91 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
92 
93 	obj = got_object_cache_get(objcache, &id);
94 	if (obj) {
95 		obj->refcnt++;
96 	} else {
97 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
98 		    objcache);
99 		if (err)
100 			goto done;
101 	}
102 
103 	err = got_privsep_send_obj(ibuf, obj);
104 done:
105 	got_object_close(obj);
106 	return err;
107 }
108 
109 static const struct got_error *
110 open_commit(struct got_commit_object **commit, struct got_pack *pack,
111     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
112     struct got_object_cache *objcache)
113 {
114 	const struct got_error *err = NULL;
115 	struct got_object *obj = NULL;
116 	uint8_t *buf = NULL;
117 	size_t len;
118 
119 	*commit = NULL;
120 
121 	obj = got_object_cache_get(objcache, id);
122 	if (obj) {
123 		obj->refcnt++;
124 	} else {
125 		err = open_object(&obj, pack, packidx, obj_idx, id,
126 		    objcache);
127 		if (err)
128 			return err;
129 	}
130 
131 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
132 	if (err)
133 		goto done;
134 
135 	obj->size = len;
136 
137 	err = got_object_parse_commit(commit, buf, len);
138 done:
139 	got_object_close(obj);
140 	free(buf);
141 	return err;
142 }
143 
144 static const struct got_error *
145 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
146     struct got_packidx *packidx, struct got_object_cache *objcache)
147 {
148 	const struct got_error *err = NULL;
149 	struct got_imsg_packed_object iobj;
150 	struct got_commit_object *commit = NULL;
151 	struct got_object_id id;
152 	size_t datalen;
153 
154 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
155 	if (datalen != sizeof(iobj))
156 		return got_error(GOT_ERR_PRIVSEP_LEN);
157 	memcpy(&iobj, imsg->data, sizeof(iobj));
158 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
159 
160 	err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
161 	if (err)
162 		goto done;
163 
164 	err = got_privsep_send_commit(ibuf, commit);
165 done:
166 	if (commit)
167 		got_object_commit_close(commit);
168 	if (err) {
169 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
170 			err = NULL;
171 		else
172 			got_privsep_send_error(ibuf, err);
173 	}
174 
175 	return err;
176 }
177 
178 static const struct got_error *
179 open_tree(uint8_t **buf, struct got_pathlist_head *entries, int *nentries,
180     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
181     struct got_object_id *id, struct got_object_cache *objcache)
182 {
183 	const struct got_error *err = NULL;
184 	struct got_object *obj = NULL;
185 	size_t len;
186 
187 	*buf = NULL;
188 	*nentries = 0;
189 
190 	obj = got_object_cache_get(objcache, id);
191 	if (obj) {
192 		obj->refcnt++;
193 	} else {
194 		err = open_object(&obj, pack, packidx, obj_idx, id,
195 		    objcache);
196 		if (err)
197 			return err;
198 	}
199 
200 	err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
201 	if (err)
202 		goto done;
203 
204 	obj->size = len;
205 
206 	err = got_object_parse_tree(entries, nentries, *buf, len);
207 done:
208 	got_object_close(obj);
209 	if (err) {
210 		free(*buf);
211 		*buf = NULL;
212 	}
213 	return err;
214 }
215 
216 static const struct got_error *
217 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
218     struct got_packidx *packidx, struct got_object_cache *objcache)
219 {
220 	const struct got_error *err = NULL;
221 	struct got_imsg_packed_object iobj;
222 	struct got_pathlist_head entries;
223 	int nentries = 0;
224 	uint8_t *buf = NULL;
225 	struct got_object_id id;
226 	size_t datalen;
227 
228 	TAILQ_INIT(&entries);
229 
230 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
231 	if (datalen != sizeof(iobj))
232 		return got_error(GOT_ERR_PRIVSEP_LEN);
233 	memcpy(&iobj, imsg->data, sizeof(iobj));
234 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
235 
236 	err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
237 	    &id, objcache);
238 	if (err)
239 		return err;
240 
241 	err = got_privsep_send_tree(ibuf, &entries, nentries);
242 	got_object_parsed_tree_entries_free(&entries);
243 	free(buf);
244 	if (err) {
245 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
246 			err = NULL;
247 		else
248 			got_privsep_send_error(ibuf, err);
249 	}
250 
251 	return err;
252 }
253 
254 static const struct got_error *
255 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
256 {
257 	const struct got_error *err;
258 	struct imsg imsg;
259 	size_t datalen;
260 
261 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
262 	if (err)
263 		return err;
264 
265 	if (imsg.hdr.type != imsg_code) {
266 		err = got_error(GOT_ERR_PRIVSEP_MSG);
267 		goto done;
268 	}
269 
270 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
271 	if (datalen != 0) {
272 		err = got_error(GOT_ERR_PRIVSEP_LEN);
273 		goto done;
274 	}
275 	if (imsg.fd == -1) {
276 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
277 		goto done;
278 	}
279 
280 	*f = fdopen(imsg.fd, "w+");
281 	if (*f == NULL) {
282 		err = got_error_from_errno("fdopen");
283 		close(imsg.fd);
284 		goto done;
285 	}
286 done:
287 	imsg_free(&imsg);
288 	return err;
289 }
290 
291 static const struct got_error *
292 receive_tempfile(FILE **basefile, FILE **accumfile, struct imsg *imsg,
293     struct imsgbuf *ibuf)
294 {
295 	size_t datalen;
296 	FILE **f;
297 
298 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
299 	if (datalen != 0)
300 		return got_error(GOT_ERR_PRIVSEP_LEN);
301 
302 	if (imsg->fd == -1)
303 		return got_error(GOT_ERR_PRIVSEP_NO_FD);
304 
305 	if (*basefile == NULL)
306 		f = basefile;
307 	else if (*accumfile == NULL)
308 		f = accumfile;
309 	else
310 		return got_error(GOT_ERR_PRIVSEP_MSG);
311 
312 	*f = fdopen(imsg->fd, "w+");
313 	if (*f == NULL)
314 		return got_error_from_errno("fdopen");
315 	imsg->fd = -1;
316 
317 	return NULL;
318 }
319 
320 static const struct got_error *
321 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
322     struct got_packidx *packidx, struct got_object_cache *objcache,
323     FILE *basefile, FILE *accumfile)
324 {
325 	const struct got_error *err = NULL;
326 	struct got_imsg_packed_object iobj;
327 	struct got_object *obj = NULL;
328 	FILE *outfile = NULL;
329 	struct got_object_id id;
330 	size_t datalen;
331 	uint64_t blob_size;
332 	uint8_t *buf = NULL;
333 
334 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
335 	if (datalen != sizeof(iobj))
336 		return got_error(GOT_ERR_PRIVSEP_LEN);
337 	memcpy(&iobj, imsg->data, sizeof(iobj));
338 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
339 
340 	obj = got_object_cache_get(objcache, &id);
341 	if (obj) {
342 		obj->refcnt++;
343 	} else {
344 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
345 		    objcache);
346 		if (err)
347 			return err;
348 	}
349 
350 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
351 	if (err)
352 		goto done;
353 
354 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
355 		err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
356 		if (err)
357 			goto done;
358 	} else
359 		blob_size = obj->size;
360 
361 	if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
362 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
363 		    obj, pack);
364 	else
365 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
366 		    accumfile);
367 	if (err)
368 		goto done;
369 
370 	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
371 done:
372 	free(buf);
373 	if (outfile && fclose(outfile) == EOF && err == NULL)
374 		err = got_error_from_errno("fclose");
375 	got_object_close(obj);
376 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
377 		got_privsep_send_error(ibuf, err);
378 
379 	return err;
380 }
381 
382 static const struct got_error *
383 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
384     struct got_packidx *packidx, struct got_object_cache *objcache)
385 {
386 	const struct got_error *err = NULL;
387 	struct got_imsg_packed_object iobj;
388 	struct got_object *obj = NULL;
389 	struct got_tag_object *tag = NULL;
390 	uint8_t *buf = NULL;
391 	size_t len;
392 	struct got_object_id id;
393 	size_t datalen;
394 
395 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
396 	if (datalen != sizeof(iobj))
397 		return got_error(GOT_ERR_PRIVSEP_LEN);
398 	memcpy(&iobj, imsg->data, sizeof(iobj));
399 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
400 
401 	obj = got_object_cache_get(objcache, &id);
402 	if (obj) {
403 		obj->refcnt++;
404 	} else {
405 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
406 		    objcache);
407 		if (err)
408 			return err;
409 	}
410 
411 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
412 	if (err)
413 		goto done;
414 
415 	obj->size = len;
416 	err = got_object_parse_tag(&tag, buf, len);
417 	if (err)
418 		goto done;
419 
420 	err = got_privsep_send_tag(ibuf, tag);
421 done:
422 	free(buf);
423 	got_object_close(obj);
424 	if (tag)
425 		got_object_tag_close(tag);
426 	if (err) {
427 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
428 			err = NULL;
429 		else
430 			got_privsep_send_error(ibuf, err);
431 	}
432 
433 	return err;
434 }
435 
436 static struct got_parsed_tree_entry *
437 find_entry_by_name(struct got_pathlist_head *entries, int nentries,
438     const char *name, size_t len)
439 {
440 	struct got_pathlist_entry *pe;
441 
442 	/* Note that tree entries are sorted in strncmp() order. */
443 	TAILQ_FOREACH(pe, entries, entry) {
444 		int cmp = strncmp(pe->path, name, len);
445 		if (cmp < 0)
446 			continue;
447 		if (cmp > 0)
448 			break;
449 		if (pe->path[len] == '\0')
450 			return (struct got_parsed_tree_entry *)pe->data;
451 	}
452 	return NULL;
453 }
454 
455 static const struct got_error *
456 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
457     struct got_pathlist_head *entries1, int *nentries1,
458     struct got_pathlist_head *entries2, int *nentries2,
459     const char *path, struct got_pack *pack, struct got_packidx *packidx,
460     struct imsgbuf *ibuf, struct got_object_cache *objcache)
461 {
462 	const struct got_error *err = NULL;
463 	struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
464 	const char *seg, *s;
465 	size_t seglen;
466 
467 	*changed = 0;
468 
469 	/* We not do support comparing the root path. */
470 	if (got_path_is_root_dir(path))
471 		return got_error_path(path, GOT_ERR_BAD_PATH);
472 
473 	s = path;
474 	while (*s == '/')
475 		s++;
476 	seg = s;
477 	seglen = 0;
478 	while (*s) {
479 		if (*s != '/') {
480 			s++;
481 			seglen++;
482 			if (*s)
483 				continue;
484 		}
485 
486 		pte1 = find_entry_by_name(entries1, *nentries1, seg, seglen);
487 		if (pte1 == NULL) {
488 			err = got_error(GOT_ERR_NO_OBJ);
489 			break;
490 		}
491 
492 		pte2 = find_entry_by_name(entries2, *nentries2, seg, seglen);
493 		if (pte2 == NULL) {
494 			*changed = 1;
495 			break;
496 		}
497 
498 		if (pte1->mode != pte2->mode) {
499 			*changed = 1;
500 			break;
501 		}
502 
503 		if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
504 			*changed = 0;
505 			break;
506 		}
507 
508 		if (*s == '\0') { /* final path element */
509 			*changed = 1;
510 			break;
511 		}
512 
513 		seg = s + 1;
514 		s++;
515 		seglen = 0;
516 		if (*s) {
517 			struct got_object_id id1, id2;
518 			int idx;
519 
520 			memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
521 			idx = got_packidx_get_object_idx(packidx, &id1);
522 			if (idx == -1) {
523 				err = got_error_no_obj(&id1);
524 				break;
525 			}
526 			got_object_parsed_tree_entries_free(entries1);
527 			*nentries1 = 0;
528 			free(*buf1);
529 			*buf1 = NULL;
530 			err = open_tree(buf1, entries1, nentries1, pack,
531 			    packidx, idx, &id1, objcache);
532 			pte1 = NULL;
533 			if (err)
534 				break;
535 
536 			memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
537 			idx = got_packidx_get_object_idx(packidx, &id2);
538 			if (idx == -1) {
539 				err = got_error_no_obj(&id2);
540 				break;
541 			}
542 			got_object_parsed_tree_entries_free(entries2);
543 			*nentries2 = 0;
544 			free(*buf2);
545 			*buf2 = NULL;
546 			err = open_tree(buf2, entries2, nentries2, pack,
547 			    packidx, idx, &id2, objcache);
548 			pte2 = NULL;
549 			if (err)
550 				break;
551 		}
552 	}
553 
554 	return err;
555 }
556 
557 static const struct got_error *
558 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
559     struct imsgbuf *ibuf)
560 {
561 	const struct got_error *err;
562 	struct ibuf *wbuf;
563 	size_t i;
564 
565 	wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
566 	    sizeof(struct got_imsg_traversed_commits) +
567 	    ncommits * SHA1_DIGEST_LENGTH);
568 	if (wbuf == NULL)
569 		return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
570 
571 	if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1) {
572 		err = got_error_from_errno("imsg_add TRAVERSED_COMMITS");
573 		ibuf_free(wbuf);
574 		return err;
575 	}
576 	for (i = 0; i < ncommits; i++) {
577 		struct got_object_id *id = &commit_ids[i];
578 		if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
579 			err = got_error_from_errno(
580 			    "imsg_add TRAVERSED_COMMITS");
581 			ibuf_free(wbuf);
582 			return err;
583 		}
584 	}
585 
586 	wbuf->fd = -1;
587 	imsg_close(ibuf, wbuf);
588 
589 	return got_privsep_flush_imsg(ibuf);
590 }
591 
592 static const struct got_error *
593 send_commit_traversal_done(struct imsgbuf *ibuf)
594 {
595 	if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
596 	    NULL, 0) == -1)
597 		return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
598 
599 	return got_privsep_flush_imsg(ibuf);
600 }
601 
602 
603 static const struct got_error *
604 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
605     struct got_pack *pack, struct got_packidx *packidx,
606     struct got_object_cache *objcache)
607 {
608 	const struct got_error *err = NULL;
609 	struct got_imsg_packed_object iobj;
610 	struct got_object_qid *pid;
611 	struct got_commit_object *commit = NULL, *pcommit = NULL;
612 	struct got_pathlist_head entries, pentries;
613 	int nentries = 0, pnentries = 0;
614 	struct got_object_id id;
615 	size_t datalen, path_len;
616 	char *path = NULL;
617 	const int min_alloc = 64;
618 	int changed = 0, ncommits = 0, nallocated = 0;
619 	struct got_object_id *commit_ids = NULL;
620 
621 	TAILQ_INIT(&entries);
622 	TAILQ_INIT(&pentries);
623 
624 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
625 	if (datalen < sizeof(iobj))
626 		return got_error(GOT_ERR_PRIVSEP_LEN);
627 	memcpy(&iobj, imsg->data, sizeof(iobj));
628 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
629 
630 	path_len = datalen - sizeof(iobj) - 1;
631 	if (path_len < 0)
632 		return got_error(GOT_ERR_PRIVSEP_LEN);
633 	if (path_len > 0) {
634 		path = imsg->data + sizeof(iobj);
635 		if (path[path_len] != '\0')
636 			return got_error(GOT_ERR_PRIVSEP_LEN);
637 	}
638 
639 	nallocated = min_alloc;
640 	commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
641 	if (commit_ids == NULL)
642 		return got_error_from_errno("reallocarray");
643 
644 	do {
645 		const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
646 		int idx;
647 
648 		if (sigint_received) {
649 			err = got_error(GOT_ERR_CANCELLED);
650 			goto done;
651 		}
652 
653 		if (commit == NULL) {
654 			idx = got_packidx_get_object_idx(packidx, &id);
655 			if (idx == -1)
656 				break;
657 			err = open_commit(&commit, pack, packidx,
658 			    idx, &id, objcache);
659 			if (err) {
660 				if (err->code != GOT_ERR_NO_OBJ)
661 					goto done;
662 				err = NULL;
663 				break;
664 			}
665 		}
666 
667 		if (sizeof(struct got_imsg_traversed_commits) +
668 		    ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
669 			err = send_traversed_commits(commit_ids, ncommits,
670 			    ibuf);
671 			if (err)
672 				goto done;
673 			ncommits = 0;
674 		}
675 		ncommits++;
676 		if (ncommits > nallocated) {
677 			struct got_object_id *new;
678 			nallocated += min_alloc;
679 			new = reallocarray(commit_ids, nallocated,
680 			    sizeof(*commit_ids));
681 			if (new == NULL) {
682 				err = got_error_from_errno("reallocarray");
683 				goto done;
684 			}
685 			commit_ids = new;
686 		}
687 		memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
688 		    SHA1_DIGEST_LENGTH);
689 
690 		pid = STAILQ_FIRST(&commit->parent_ids);
691 		if (pid == NULL)
692 			break;
693 
694 		idx = got_packidx_get_object_idx(packidx, pid->id);
695 		if (idx == -1)
696 			break;
697 
698 		err = open_commit(&pcommit, pack, packidx, idx, pid->id,
699 		    objcache);
700 		if (err) {
701 			if (err->code != GOT_ERR_NO_OBJ)
702 				goto done;
703 			err = NULL;
704 			break;
705 		}
706 
707 		if (path[0] == '/' && path[1] == '\0') {
708 			if (got_object_id_cmp(pcommit->tree_id,
709 			    commit->tree_id) != 0) {
710 				changed = 1;
711 				break;
712 			}
713 		} else {
714 			int pidx;
715 			uint8_t *buf = NULL, *pbuf = NULL;
716 
717 			idx = got_packidx_get_object_idx(packidx,
718 			    commit->tree_id);
719 			if (idx == -1)
720 				break;
721 			pidx = got_packidx_get_object_idx(packidx,
722 			    pcommit->tree_id);
723 			if (pidx == -1)
724 				break;
725 
726 			err = open_tree(&buf, &entries, &nentries, pack,
727 			    packidx, idx, commit->tree_id, objcache);
728 			if (err)
729 				goto done;
730 			err = open_tree(&pbuf, &pentries, &pnentries, pack,
731 			    packidx, pidx, pcommit->tree_id, objcache);
732 			if (err) {
733 				free(buf);
734 				goto done;
735 			}
736 
737 			err = tree_path_changed(&changed, &buf, &pbuf,
738 			    &entries, &nentries, &pentries, &pnentries, path,
739 			    pack, packidx, ibuf, objcache);
740 
741 			got_object_parsed_tree_entries_free(&entries);
742 			nentries = 0;
743 			free(buf);
744 			got_object_parsed_tree_entries_free(&pentries);
745 			pnentries = 0;
746 			free(pbuf);
747 			if (err) {
748 				if (err->code != GOT_ERR_NO_OBJ)
749 					goto done;
750 				err = NULL;
751 				break;
752 			}
753 		}
754 
755 		if (!changed) {
756 			memcpy(id.sha1, pid->id->sha1, SHA1_DIGEST_LENGTH);
757 			got_object_commit_close(commit);
758 			commit = pcommit;
759 			pcommit = NULL;
760 		}
761 	} while (!changed);
762 
763 	if (ncommits > 0) {
764 		err = send_traversed_commits(commit_ids, ncommits, ibuf);
765 		if (err)
766 			goto done;
767 
768 		if (changed) {
769 			err = got_privsep_send_commit(ibuf, commit);
770 			if (err)
771 				goto done;
772 		}
773 	}
774 	err = send_commit_traversal_done(ibuf);
775 done:
776 	free(commit_ids);
777 	if (commit)
778 		got_object_commit_close(commit);
779 	if (pcommit)
780 		got_object_commit_close(pcommit);
781 	if (nentries != 0)
782 		got_object_parsed_tree_entries_free(&entries);
783 	if (pnentries != 0)
784 		got_object_parsed_tree_entries_free(&pentries);
785 	if (err) {
786 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
787 			err = NULL;
788 		else
789 			got_privsep_send_error(ibuf, err);
790 	}
791 
792 	return err;
793 }
794 
795 static const struct got_error *
796 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
797     struct got_pack *pack, struct got_packidx *packidx,
798     struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
799 {
800 	const struct got_error *err = NULL;
801 	uint8_t *buf = NULL;
802 	uint64_t size = 0;
803 	FILE *outfile = NULL;
804 	struct got_imsg_packed_object iobj;
805 	struct got_object *obj;
806 	struct got_object_id id;
807 	size_t datalen;
808 
809 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
810 	if (datalen != sizeof(iobj))
811 		return got_error(GOT_ERR_PRIVSEP_LEN);
812 	memcpy(&iobj, imsg->data, sizeof(iobj));
813 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
814 
815 	obj = got_object_cache_get(objcache, &id);
816 	if (obj) {
817 		obj->refcnt++;
818 	} else {
819 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
820 		    objcache);
821 		if (err)
822 			return err;
823 	}
824 
825 	err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
826 	if (err)
827 		return err;
828 
829 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
830 		err = got_pack_get_max_delta_object_size(&size, obj, pack);
831 		if (err)
832 			goto done;
833 	} else
834 		size = obj->size;
835 
836 	if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
837 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
838 		    obj, pack);
839 	else
840 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
841 		    accumfile);
842 	if (err)
843 		goto done;
844 
845 	err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
846 done:
847 	free(buf);
848 	if (outfile && fclose(outfile) == EOF && err == NULL)
849 		err = got_error_from_errno("fclose");
850 	got_object_close(obj);
851 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
852 		got_privsep_send_error(ibuf, err);
853 
854 	return err;
855 }
856 
857 
858 
859 static const struct got_error *
860 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
861 {
862 	const struct got_error *err = NULL;
863 	struct imsg imsg;
864 	struct got_imsg_packidx ipackidx;
865 	size_t datalen;
866 	struct got_packidx *p;
867 
868 	*packidx = NULL;
869 
870 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
871 	if (err)
872 		return err;
873 
874 	p = calloc(1, sizeof(*p));
875 	if (p == NULL) {
876 		err = got_error_from_errno("calloc");
877 		goto done;
878 	}
879 
880 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
881 		err = got_error(GOT_ERR_PRIVSEP_MSG);
882 		goto done;
883 	}
884 
885 	if (imsg.fd == -1) {
886 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
887 		goto done;
888 	}
889 
890 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
891 	if (datalen != sizeof(ipackidx)) {
892 		err = got_error(GOT_ERR_PRIVSEP_LEN);
893 		goto done;
894 	}
895 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
896 
897 	p->len = ipackidx.len;
898 	p->fd = dup(imsg.fd);
899 	if (p->fd == -1) {
900 		err = got_error_from_errno("dup");
901 		goto done;
902 	}
903 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
904 		err = got_error_from_errno("lseek");
905 		goto done;
906 	}
907 
908 #ifndef GOT_PACK_NO_MMAP
909 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
910 	if (p->map == MAP_FAILED)
911 		p->map = NULL; /* fall back to read(2) */
912 #endif
913 	err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
914 done:
915 	if (err) {
916 		if (imsg.fd != -1)
917 			close(imsg.fd);
918 		got_packidx_close(p);
919 	} else
920 		*packidx = p;
921 	imsg_free(&imsg);
922 	return err;
923 }
924 
925 static const struct got_error *
926 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
927 {
928 	const struct got_error *err = NULL;
929 	struct imsg imsg;
930 	struct got_imsg_pack ipack;
931 	size_t datalen;
932 	struct got_pack *pack;
933 
934 	*packp = NULL;
935 
936 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
937 	if (err)
938 		return err;
939 
940 	pack = calloc(1, sizeof(*pack));
941 	if (pack == NULL) {
942 		err = got_error_from_errno("calloc");
943 		goto done;
944 	}
945 
946 	if (imsg.hdr.type != GOT_IMSG_PACK) {
947 		err = got_error(GOT_ERR_PRIVSEP_MSG);
948 		goto done;
949 	}
950 
951 	if (imsg.fd == -1) {
952 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
953 		goto done;
954 	}
955 
956 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
957 	if (datalen != sizeof(ipack)) {
958 		err = got_error(GOT_ERR_PRIVSEP_LEN);
959 		goto done;
960 	}
961 	memcpy(&ipack, imsg.data, sizeof(ipack));
962 
963 	pack->filesize = ipack.filesize;
964 	pack->fd = dup(imsg.fd);
965 	if (pack->fd == -1) {
966 		err = got_error_from_errno("dup");
967 		goto done;
968 	}
969 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
970 		err = got_error_from_errno("lseek");
971 		goto done;
972 	}
973 	pack->path_packfile = strdup(ipack.path_packfile);
974 	if (pack->path_packfile == NULL) {
975 		err = got_error_from_errno("strdup");
976 		goto done;
977 	}
978 
979 	pack->delta_cache = got_delta_cache_alloc(100,
980 	    GOT_DELTA_RESULT_SIZE_CACHED_MAX);
981 	if (pack->delta_cache == NULL) {
982 		err = got_error_from_errno("got_delta_cache_alloc");
983 		goto done;
984 	}
985 
986 #ifndef GOT_PACK_NO_MMAP
987 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
988 	    pack->fd, 0);
989 	if (pack->map == MAP_FAILED)
990 		pack->map = NULL; /* fall back to read(2) */
991 #endif
992 done:
993 	if (err) {
994 		if (imsg.fd != -1)
995 			close(imsg.fd);
996 		free(pack);
997 	} else
998 		*packp = pack;
999 	imsg_free(&imsg);
1000 	return err;
1001 }
1002 
1003 int
1004 main(int argc, char *argv[])
1005 {
1006 	const struct got_error *err = NULL;
1007 	struct imsgbuf ibuf;
1008 	struct imsg imsg;
1009 	struct got_packidx *packidx = NULL;
1010 	struct got_pack *pack = NULL;
1011 	struct got_object_cache objcache;
1012 	FILE *basefile = NULL, *accumfile = NULL;
1013 
1014 	//static int attached;
1015 	//while (!attached) sleep(1);
1016 
1017 	signal(SIGINT, catch_sigint);
1018 
1019 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1020 
1021 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1022 	if (err) {
1023 		err = got_error_from_errno("got_object_cache_init");
1024 		got_privsep_send_error(&ibuf, err);
1025 		return 1;
1026 	}
1027 
1028 #ifndef PROFILE
1029 	/* revoke access to most system calls */
1030 	if (pledge("stdio recvfd", NULL) == -1) {
1031 		err = got_error_from_errno("pledge");
1032 		got_privsep_send_error(&ibuf, err);
1033 		return 1;
1034 	}
1035 #endif
1036 
1037 	err = receive_packidx(&packidx, &ibuf);
1038 	if (err) {
1039 		got_privsep_send_error(&ibuf, err);
1040 		return 1;
1041 	}
1042 
1043 	err = receive_pack(&pack, &ibuf);
1044 	if (err) {
1045 		got_privsep_send_error(&ibuf, err);
1046 		return 1;
1047 	}
1048 
1049 	for (;;) {
1050 		imsg.fd = -1;
1051 
1052 		if (sigint_received) {
1053 			err = got_error(GOT_ERR_CANCELLED);
1054 			break;
1055 		}
1056 
1057 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1058 		if (err) {
1059 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
1060 				err = NULL;
1061 			break;
1062 		}
1063 
1064 		if (imsg.hdr.type == GOT_IMSG_STOP)
1065 			break;
1066 
1067 		switch (imsg.hdr.type) {
1068 		case GOT_IMSG_TMPFD:
1069 			err = receive_tempfile(&basefile, &accumfile,
1070 			    &imsg, &ibuf);
1071 			break;
1072 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
1073 			err = object_request(&imsg, &ibuf, pack, packidx,
1074 			    &objcache);
1075 			break;
1076 		case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1077 			if (basefile == NULL || accumfile == NULL) {
1078 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1079 				break;
1080 			}
1081 			err = raw_object_request(&imsg, &ibuf, pack, packidx,
1082 			    &objcache, basefile, accumfile);
1083 			break;
1084 		case GOT_IMSG_COMMIT_REQUEST:
1085 			err = commit_request(&imsg, &ibuf, pack, packidx,
1086 			    &objcache);
1087 			break;
1088 		case GOT_IMSG_TREE_REQUEST:
1089 			err = tree_request(&imsg, &ibuf, pack, packidx,
1090 			    &objcache);
1091 			break;
1092 		case GOT_IMSG_BLOB_REQUEST:
1093 			if (basefile == NULL || accumfile == NULL) {
1094 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1095 				break;
1096 			}
1097 			err = blob_request(&imsg, &ibuf, pack, packidx,
1098 			    &objcache, basefile, accumfile);
1099 			break;
1100 		case GOT_IMSG_TAG_REQUEST:
1101 			err = tag_request(&imsg, &ibuf, pack, packidx,
1102 			    &objcache);
1103 			break;
1104 		case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1105 			err = commit_traversal_request(&imsg, &ibuf, pack,
1106 			    packidx, &objcache);
1107 			break;
1108 		default:
1109 			err = got_error(GOT_ERR_PRIVSEP_MSG);
1110 			break;
1111 		}
1112 
1113 		if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1114 			err = got_error_from_errno("close");
1115 		imsg_free(&imsg);
1116 		if (err)
1117 			break;
1118 	}
1119 
1120 	if (packidx)
1121 		got_packidx_close(packidx);
1122 	if (pack)
1123 		got_pack_close(pack);
1124 	got_object_cache_close(&objcache);
1125 	imsg_clear(&ibuf);
1126 	if (basefile && fclose(basefile) == EOF && err == NULL)
1127 		err = got_error_from_errno("fclose");
1128 	if (accumfile && fclose(accumfile) == EOF && err == NULL)
1129 		err = got_error_from_errno("fclose");
1130 	if (err) {
1131 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1132 			fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1133 			got_privsep_send_error(&ibuf, err);
1134 		}
1135 	}
1136 	if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1137 		err = got_error_from_errno("close");
1138 	return err ? 1 : 0;
1139 }