Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16 #include "got_compat.h"
18 #include <sys/stat.h>
19 #include <sys/types.h>
20 #include <sys/queue.h>
21 #include <sys/uio.h>
22 #include <sys/time.h>
23 #include <sys/mman.h>
25 #include <inttypes.h>
26 #include <limits.h>
27 #include <signal.h>
28 #include <stdint.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <zlib.h>
35 #include "got_error.h"
36 #include "got_object.h"
37 #include "got_path.h"
39 #include "got_lib_delta.h"
40 #include "got_lib_delta_cache.h"
41 #include "got_lib_object.h"
42 #include "got_lib_object_qid.h"
43 #include "got_lib_object_cache.h"
44 #include "got_lib_object_parse.h"
45 #include "got_lib_object_idset.h"
46 #include "got_lib_privsep.h"
47 #include "got_lib_pack.h"
49 #ifndef nitems
50 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
51 #endif
53 static volatile sig_atomic_t sigint_received;
55 static void
56 catch_sigint(int signo)
57 {
58 sigint_received = 1;
59 }
61 static const struct got_error *
62 open_object(struct got_object **obj, struct got_pack *pack,
63 struct got_packidx *packidx, int idx, struct got_object_id *id,
64 struct got_object_cache *objcache)
65 {
66 const struct got_error *err;
68 err = got_packfile_open_object(obj, pack, packidx, idx, id);
69 if (err)
70 return err;
71 (*obj)->refcnt++;
73 err = got_object_cache_add(objcache, id, *obj);
74 if (err) {
75 if (err->code == GOT_ERR_OBJ_EXISTS ||
76 err->code == GOT_ERR_OBJ_TOO_LARGE)
77 err = NULL;
78 return err;
79 }
80 (*obj)->refcnt++;
81 return NULL;
82 }
84 static const struct got_error *
85 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
86 struct got_packidx *packidx, struct got_object_cache *objcache)
87 {
88 const struct got_error *err = NULL;
89 struct got_imsg_packed_object iobj;
90 struct got_object *obj;
91 struct got_object_id id;
92 size_t datalen;
94 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
95 if (datalen != sizeof(iobj))
96 return got_error(GOT_ERR_PRIVSEP_LEN);
97 memcpy(&iobj, imsg->data, sizeof(iobj));
98 memcpy(&id, &iobj.id, sizeof(id));
100 obj = got_object_cache_get(objcache, &id);
101 if (obj) {
102 obj->refcnt++;
103 } else {
104 err = open_object(&obj, pack, packidx, iobj.idx, &id,
105 objcache);
106 if (err)
107 goto done;
110 err = got_privsep_send_obj(ibuf, obj);
111 done:
112 got_object_close(obj);
113 return err;
116 static const struct got_error *
117 open_commit(struct got_commit_object **commit, struct got_pack *pack,
118 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
119 struct got_object_cache *objcache)
121 const struct got_error *err = NULL;
122 struct got_object *obj = NULL;
123 uint8_t *buf = NULL;
124 size_t len;
126 *commit = NULL;
128 obj = got_object_cache_get(objcache, id);
129 if (obj) {
130 obj->refcnt++;
131 } else {
132 err = open_object(&obj, pack, packidx, obj_idx, id,
133 objcache);
134 if (err)
135 return err;
138 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
139 if (err)
140 goto done;
142 obj->size = len;
144 err = got_object_parse_commit(commit, buf, len);
145 done:
146 got_object_close(obj);
147 free(buf);
148 return err;
151 static const struct got_error *
152 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
153 struct got_packidx *packidx, struct got_object_cache *objcache)
155 const struct got_error *err = NULL;
156 struct got_imsg_packed_object iobj;
157 struct got_commit_object *commit = NULL;
158 struct got_object_id id;
159 size_t datalen;
161 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
162 if (datalen != sizeof(iobj))
163 return got_error(GOT_ERR_PRIVSEP_LEN);
164 memcpy(&iobj, imsg->data, sizeof(iobj));
165 memcpy(&id, &iobj.id, sizeof(id));
167 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
168 if (err)
169 goto done;
171 err = got_privsep_send_commit(ibuf, commit);
172 done:
173 if (commit)
174 got_object_commit_close(commit);
175 if (err) {
176 if (err->code == GOT_ERR_PRIVSEP_PIPE)
177 err = NULL;
178 else
179 got_privsep_send_error(ibuf, err);
182 return err;
185 static const struct got_error *
186 open_tree(uint8_t **buf, size_t *len,
187 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
188 struct got_object_id *id, struct got_object_cache *objcache)
190 const struct got_error *err = NULL;
191 struct got_object *obj = NULL;
192 int cached = 0;
194 *buf = NULL;
195 *len = 0;
197 obj = got_object_cache_get(objcache, id);
198 if (obj) {
199 obj->refcnt++;
200 cached = 1;
201 } else {
202 err = open_object(&obj, pack, packidx, obj_idx, id,
203 objcache);
204 if (err)
205 return err;
208 err = got_packfile_extract_object_to_mem(buf, len, obj, pack);
209 if (err)
210 goto done;
212 if (!cached)
213 obj->size = *len;
214 done:
215 got_object_close(obj);
216 if (err) {
217 free(*buf);
218 *buf = NULL;
220 return err;
223 static const struct got_error *
224 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
225 struct got_packidx *packidx, struct got_object_cache *objcache,
226 struct got_parsed_tree_entry **entries, size_t *nentries,
227 size_t *nentries_alloc)
229 const struct got_error *err = NULL;
230 struct got_imsg_packed_object iobj;
231 uint8_t *buf = NULL;
232 size_t len = 0;
233 struct got_object_id id;
234 size_t datalen;
236 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
237 if (datalen != sizeof(iobj))
238 return got_error(GOT_ERR_PRIVSEP_LEN);
239 memcpy(&iobj, imsg->data, sizeof(iobj));
240 memcpy(&id, &iobj.id, sizeof(id));
242 err = open_tree(&buf, &len, pack, packidx, iobj.idx, &id, objcache);
243 if (err)
244 return err;
246 err = got_object_parse_tree(entries, nentries, nentries_alloc,
247 buf, len);
248 if (err)
249 goto done;
251 err = got_privsep_send_tree(ibuf, *entries, *nentries);
252 if (err) {
253 if (err->code == GOT_ERR_PRIVSEP_PIPE)
254 err = NULL;
255 else
256 got_privsep_send_error(ibuf, err);
258 done:
259 free(buf);
260 return err;
263 static const struct got_error *
264 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
266 const struct got_error *err;
267 struct imsg imsg;
268 size_t datalen;
269 int fd;
271 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
272 if (err)
273 return err;
275 if (imsg.hdr.type != imsg_code) {
276 err = got_error(GOT_ERR_PRIVSEP_MSG);
277 goto done;
280 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
281 if (datalen != 0) {
282 err = got_error(GOT_ERR_PRIVSEP_LEN);
283 goto done;
285 fd = imsg_get_fd(&imsg);
286 if (fd == -1) {
287 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
288 goto done;
291 *f = fdopen(fd, "w+");
292 if (*f == NULL) {
293 err = got_error_from_errno("fdopen");
294 close(fd);
295 goto done;
297 done:
298 imsg_free(&imsg);
299 return err;
302 static const struct got_error *
303 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
304 struct imsgbuf *ibuf)
306 const struct got_error *err;
307 size_t datalen;
308 int fd;
310 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
311 if (datalen != 0)
312 return got_error(GOT_ERR_PRIVSEP_LEN);
314 fd = imsg_get_fd(imsg);
315 if (fd == -1)
316 return got_error(GOT_ERR_PRIVSEP_NO_FD);
318 *f = fdopen(fd, mode);
319 if (*f == NULL) {
320 err = got_error_from_errno("fdopen");
321 close(fd);
322 return err;
325 return NULL;
328 static const struct got_error *
329 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
330 struct got_packidx *packidx, struct got_object_cache *objcache,
331 FILE *basefile, FILE *accumfile)
333 const struct got_error *err = NULL;
334 struct got_imsg_packed_object iobj;
335 struct got_object *obj = NULL;
336 FILE *outfile = NULL;
337 struct got_object_id id;
338 size_t datalen;
339 uint64_t blob_size;
340 uint8_t *buf = NULL;
342 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
343 if (datalen != sizeof(iobj))
344 return got_error(GOT_ERR_PRIVSEP_LEN);
345 memcpy(&iobj, imsg->data, sizeof(iobj));
346 memcpy(&id, &iobj.id, sizeof(id));
348 obj = got_object_cache_get(objcache, &id);
349 if (obj) {
350 obj->refcnt++;
351 } else {
352 err = open_object(&obj, pack, packidx, iobj.idx, &id,
353 objcache);
354 if (err)
355 return err;
358 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
359 if (err)
360 goto done;
362 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
363 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
364 if (err)
365 goto done;
366 } else
367 blob_size = obj->size;
369 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
370 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
371 obj, pack);
372 else
373 err = got_packfile_extract_object(pack, obj, outfile, basefile,
374 accumfile);
375 if (err)
376 goto done;
378 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
379 done:
380 free(buf);
381 if (outfile && fclose(outfile) == EOF && err == NULL)
382 err = got_error_from_errno("fclose");
383 got_object_close(obj);
384 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
385 got_privsep_send_error(ibuf, err);
387 return err;
390 static const struct got_error *
391 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
392 struct got_packidx *packidx, struct got_object_cache *objcache)
394 const struct got_error *err = NULL;
395 struct got_imsg_packed_object iobj;
396 struct got_object *obj = NULL;
397 struct got_tag_object *tag = NULL;
398 uint8_t *buf = NULL;
399 size_t len;
400 struct got_object_id id;
401 size_t datalen;
403 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
404 if (datalen != sizeof(iobj))
405 return got_error(GOT_ERR_PRIVSEP_LEN);
406 memcpy(&iobj, imsg->data, sizeof(iobj));
407 memcpy(&id, &iobj.id, sizeof(id));
409 obj = got_object_cache_get(objcache, &id);
410 if (obj) {
411 obj->refcnt++;
412 } else {
413 err = open_object(&obj, pack, packidx, iobj.idx, &id,
414 objcache);
415 if (err)
416 return err;
419 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
420 if (err)
421 goto done;
423 obj->size = len;
424 err = got_object_parse_tag(&tag, buf, len);
425 if (err)
426 goto done;
428 err = got_privsep_send_tag(ibuf, tag);
429 done:
430 free(buf);
431 got_object_close(obj);
432 if (tag)
433 got_object_tag_close(tag);
434 if (err) {
435 if (err->code == GOT_ERR_PRIVSEP_PIPE)
436 err = NULL;
437 else
438 got_privsep_send_error(ibuf, err);
441 return err;
444 static const struct got_error *
445 tree_path_changed(int *changed, uint8_t **buf1, size_t *len1,
446 uint8_t **buf2, size_t *len2, const char *path,
447 struct got_pack *pack, struct got_packidx *packidx,
448 struct imsgbuf *ibuf, struct got_object_cache *objcache)
450 const struct got_error *err = NULL;
451 struct got_parsed_tree_entry pte1, pte2;
452 const char *seg, *s;
453 size_t seglen;
454 size_t remain1 = *len1, remain2 = *len2, elen;
455 uint8_t *next_entry1 = *buf1;
456 uint8_t *next_entry2 = *buf2;
458 memset(&pte1, 0, sizeof(pte1));
459 memset(&pte2, 0, sizeof(pte2));
461 *changed = 0;
463 /* We not do support comparing the root path. */
464 if (got_path_is_root_dir(path))
465 return got_error_path(path, GOT_ERR_BAD_PATH);
467 s = path;
468 while (*s == '/')
469 s++;
470 seg = s;
471 seglen = 0;
472 while (*s) {
473 if (*s != '/') {
474 s++;
475 seglen++;
476 if (*s)
477 continue;
480 /*
481 * As an optimization we compare entries in on-disk order
482 * rather than in got_path_cmp() order. We only need to
483 * find out if any entries differ. Parsing all entries and
484 * sorting them slows us down significantly when tree objects
485 * have thousands of entries. We can assume that on-disk entry
486 * ordering is stable, as per got_object_tree_create() and
487 * sort_tree_entries_the_way_git_likes_it(). Other orderings
488 * are incompatible with Git and would yield false positives
489 * here, too.
490 */
491 while (remain1 > 0) {
492 err = got_object_parse_tree_entry(&pte1, &elen,
493 next_entry1, remain1);
494 if (err)
495 return err;
496 next_entry1 += elen;
497 remain1 -= elen;
498 if (strncmp(pte1.name, seg, seglen) != 0 ||
499 pte1.name[seglen] != '\0') {
500 memset(&pte1, 0, sizeof(pte1));
501 continue;
502 } else
503 break;
505 if (pte1.name == NULL) {
506 err = got_error(GOT_ERR_NO_OBJ);
507 break;
510 if (remain2 == 0) {
511 *changed = 1;
512 break;
515 while (remain2 > 0) {
516 err = got_object_parse_tree_entry(&pte2, &elen,
517 next_entry2, remain2);
518 if (err)
519 return err;
520 next_entry2 += elen;
521 remain2 -= elen;
522 if (strncmp(pte2.name, seg, seglen) != 0 ||
523 pte2.name[seglen] != '\0') {
524 memset(&pte2, 0, sizeof(pte2));
525 continue;
526 } else
527 break;
530 if (pte2.name == NULL) {
531 *changed = 1;
532 break;
535 if (pte1.mode != pte2.mode) {
536 *changed = 1;
537 break;
540 if (memcmp(pte1.id, pte2.id, SHA1_DIGEST_LENGTH) == 0) {
541 *changed = 0;
542 break;
545 if (*s == '\0') { /* final path element */
546 *changed = 1;
547 break;
550 seg = s + 1;
551 s++;
552 seglen = 0;
553 if (*s) {
554 struct got_object_id id1, id2;
555 int idx;
557 memcpy(id1.sha1, pte1.id, SHA1_DIGEST_LENGTH);
558 idx = got_packidx_get_object_idx(packidx, &id1);
559 if (idx == -1) {
560 err = got_error_no_obj(&id1);
561 break;
563 free(*buf1);
564 *buf1 = NULL;
565 err = open_tree(buf1, len1, pack, packidx, idx, &id1,
566 objcache);
567 memset(&pte1, 0, sizeof(pte1));
568 if (err)
569 break;
570 next_entry1 = *buf1;
571 remain1 = *len1;
573 memcpy(id2.sha1, pte2.id, SHA1_DIGEST_LENGTH);
574 idx = got_packidx_get_object_idx(packidx, &id2);
575 if (idx == -1) {
576 err = got_error_no_obj(&id2);
577 break;
579 free(*buf2);
580 *buf2 = NULL;
581 err = open_tree(buf2, len2, pack, packidx, idx, &id2,
582 objcache);
583 memset(&pte2, 0, sizeof(pte2));
584 if (err)
585 break;
586 next_entry2 = *buf2;
587 remain2 = *len2;
591 return err;
594 static const struct got_error *
595 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
596 struct imsgbuf *ibuf)
598 struct ibuf *wbuf;
599 size_t i;
601 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
602 sizeof(struct got_imsg_traversed_commits) +
603 ncommits * sizeof(commit_ids[0]));
604 if (wbuf == NULL)
605 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
607 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
608 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
610 for (i = 0; i < ncommits; i++) {
611 struct got_object_id *id = &commit_ids[i];
612 if (imsg_add(wbuf, id, sizeof(*id)) == -1) {
613 return got_error_from_errno(
614 "imsg_add TRAVERSED_COMMITS");
618 imsg_close(ibuf, wbuf);
620 return got_privsep_flush_imsg(ibuf);
623 static const struct got_error *
624 send_commit_traversal_done(struct imsgbuf *ibuf)
626 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
627 NULL, 0) == -1)
628 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
630 return got_privsep_flush_imsg(ibuf);
633 static const struct got_error *
634 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
635 struct got_pack *pack, struct got_packidx *packidx,
636 struct got_object_cache *objcache)
638 const struct got_error *err = NULL;
639 struct got_imsg_commit_traversal_request ctreq;
640 struct got_object_qid *pid;
641 struct got_commit_object *commit = NULL, *pcommit = NULL;
642 struct got_object_id id;
643 size_t datalen;
644 char *path = NULL;
645 const int min_alloc = 64;
646 int changed = 0, ncommits = 0, nallocated = 0;
647 struct got_object_id *commit_ids = NULL;
649 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
650 if (datalen < sizeof(ctreq))
651 return got_error(GOT_ERR_PRIVSEP_LEN);
652 memcpy(&ctreq, imsg->data, sizeof(ctreq));
653 memcpy(&id, &ctreq.iobj.id, sizeof(id));
655 if (datalen != sizeof(ctreq) + ctreq.path_len)
656 return got_error(GOT_ERR_PRIVSEP_LEN);
657 if (ctreq.path_len == 0)
658 return got_error(GOT_ERR_PRIVSEP_LEN);
660 path = strndup(imsg->data + sizeof(ctreq), ctreq.path_len);
661 if (path == NULL)
662 return got_error_from_errno("strndup");
664 nallocated = min_alloc;
665 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
666 if (commit_ids == NULL)
667 return got_error_from_errno("reallocarray");
669 do {
670 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
671 int idx;
673 if (sigint_received) {
674 err = got_error(GOT_ERR_CANCELLED);
675 goto done;
678 if (commit == NULL) {
679 idx = got_packidx_get_object_idx(packidx, &id);
680 if (idx == -1)
681 break;
682 err = open_commit(&commit, pack, packidx,
683 idx, &id, objcache);
684 if (err) {
685 if (err->code != GOT_ERR_NO_OBJ)
686 goto done;
687 err = NULL;
688 break;
692 if (sizeof(struct got_imsg_traversed_commits) +
693 ncommits * sizeof(commit_ids[0]) >= max_datalen) {
694 err = send_traversed_commits(commit_ids, ncommits,
695 ibuf);
696 if (err)
697 goto done;
698 ncommits = 0;
700 ncommits++;
701 if (ncommits > nallocated) {
702 struct got_object_id *new;
703 nallocated += min_alloc;
704 new = reallocarray(commit_ids, nallocated,
705 sizeof(*commit_ids));
706 if (new == NULL) {
707 err = got_error_from_errno("reallocarray");
708 goto done;
710 commit_ids = new;
712 memcpy(&commit_ids[ncommits - 1], &id, sizeof(id));
714 pid = STAILQ_FIRST(&commit->parent_ids);
715 if (pid == NULL)
716 break;
718 idx = got_packidx_get_object_idx(packidx, &pid->id);
719 if (idx == -1)
720 break;
722 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
723 objcache);
724 if (err) {
725 if (err->code != GOT_ERR_NO_OBJ)
726 goto done;
727 err = NULL;
728 break;
731 if (path[0] == '/' && path[1] == '\0') {
732 if (got_object_id_cmp(pcommit->tree_id,
733 commit->tree_id) != 0) {
734 changed = 1;
735 break;
737 } else {
738 int pidx;
739 uint8_t *buf = NULL, *pbuf = NULL;
740 size_t len = 0, plen = 0;
742 idx = got_packidx_get_object_idx(packidx,
743 commit->tree_id);
744 if (idx == -1)
745 break;
746 pidx = got_packidx_get_object_idx(packidx,
747 pcommit->tree_id);
748 if (pidx == -1)
749 break;
751 err = open_tree(&buf, &len, pack, packidx, idx,
752 commit->tree_id, objcache);
753 if (err)
754 goto done;
756 err = open_tree(&pbuf, &plen, pack, packidx, pidx,
757 pcommit->tree_id, objcache);
758 if (err) {
759 free(buf);
760 goto done;
763 err = tree_path_changed(&changed, &buf, &len,
764 &pbuf, &plen, path, pack, packidx, ibuf,
765 objcache);
767 free(buf);
768 free(pbuf);
769 if (err) {
770 if (err->code != GOT_ERR_NO_OBJ)
771 goto done;
772 err = NULL;
773 break;
777 if (!changed) {
778 memcpy(&id, &pid->id, sizeof(id));
779 got_object_commit_close(commit);
780 commit = pcommit;
781 pcommit = NULL;
783 } while (!changed);
785 if (ncommits > 0) {
786 err = send_traversed_commits(commit_ids, ncommits, ibuf);
787 if (err)
788 goto done;
790 if (changed) {
791 err = got_privsep_send_commit(ibuf, commit);
792 if (err)
793 goto done;
796 err = send_commit_traversal_done(ibuf);
797 done:
798 free(path);
799 free(commit_ids);
800 if (commit)
801 got_object_commit_close(commit);
802 if (pcommit)
803 got_object_commit_close(pcommit);
804 if (err) {
805 if (err->code == GOT_ERR_PRIVSEP_PIPE)
806 err = NULL;
807 else
808 got_privsep_send_error(ibuf, err);
811 return err;
814 static const struct got_error *
815 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
816 struct got_pack *pack, struct got_packidx *packidx,
817 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
819 const struct got_error *err = NULL;
820 uint8_t *buf = NULL;
821 uint64_t size = 0;
822 FILE *outfile = NULL;
823 struct got_imsg_packed_object iobj;
824 struct got_object *obj;
825 struct got_object_id id;
826 size_t datalen;
828 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
829 if (datalen != sizeof(iobj))
830 return got_error(GOT_ERR_PRIVSEP_LEN);
831 memcpy(&iobj, imsg->data, sizeof(iobj));
832 memcpy(&id, &iobj.id, sizeof(id));
834 obj = got_object_cache_get(objcache, &id);
835 if (obj) {
836 obj->refcnt++;
837 } else {
838 err = open_object(&obj, pack, packidx, iobj.idx, &id,
839 objcache);
840 if (err)
841 return err;
844 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
845 if (err)
846 return err;
848 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
849 err = got_pack_get_max_delta_object_size(&size, obj, pack);
850 if (err)
851 goto done;
852 } else
853 size = obj->size;
855 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
856 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
857 obj, pack);
858 else
859 err = got_packfile_extract_object(pack, obj, outfile, basefile,
860 accumfile);
861 if (err)
862 goto done;
864 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
865 done:
866 free(buf);
867 if (outfile && fclose(outfile) == EOF && err == NULL)
868 err = got_error_from_errno("fclose");
869 got_object_close(obj);
870 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
871 got_privsep_send_error(ibuf, err);
873 return err;
876 static const struct got_error *
877 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
878 off_t base_offset)
880 const struct got_error *err;
881 int idx;
883 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
884 if (err)
885 return err;
886 if (idx == -1)
887 return got_error(GOT_ERR_BAD_PACKIDX);
889 return got_packidx_get_object_id(base_id, packidx, idx);
892 static const struct got_error *
893 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
894 FILE *delta_outfile, struct got_pack *pack,
895 struct got_packidx *packidx)
897 const struct got_error *err = NULL;
898 struct got_imsg_raw_delta_request req;
899 size_t datalen, delta_size, delta_compressed_size;
900 off_t delta_offset, delta_data_offset;
901 uint8_t *delta_buf = NULL;
902 struct got_object_id id, base_id;
903 off_t base_offset, delta_out_offset = 0;
904 uint64_t base_size = 0, result_size = 0;
905 size_t w;
907 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
908 if (datalen != sizeof(req))
909 return got_error(GOT_ERR_PRIVSEP_LEN);
910 memcpy(&req, imsg->data, sizeof(req));
911 memcpy(&id, &req.id, sizeof(id));
913 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
914 &delta_compressed_size, &delta_offset, &delta_data_offset,
915 &base_offset, &base_id, &base_size, &result_size,
916 pack, packidx, req.idx);
917 if (err)
918 goto done;
920 /*
921 * If this is an offset delta we must determine the base
922 * object ID ourselves.
923 */
924 if (base_offset != 0) {
925 err = get_base_object_id(&base_id, packidx, base_offset);
926 if (err)
927 goto done;
930 delta_out_offset = ftello(delta_outfile);
931 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
932 if (w != delta_compressed_size) {
933 err = got_ferror(delta_outfile, GOT_ERR_IO);
934 goto done;
936 if (fflush(delta_outfile) == -1) {
937 err = got_error_from_errno("fflush");
938 goto done;
941 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
942 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
943 &base_id);
944 done:
945 free(delta_buf);
946 return err;
949 struct search_deltas_arg {
950 struct imsgbuf *ibuf;
951 struct got_packidx *packidx;
952 struct got_pack *pack;
953 struct got_object_idset *idset;
954 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
955 size_t ndeltas;
956 };
958 static const struct got_error *
959 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
961 const struct got_error *err;
962 struct search_deltas_arg *a = arg;
963 int obj_idx;
964 uint8_t *delta_buf = NULL;
965 uint64_t base_size, result_size;
966 size_t delta_size, delta_compressed_size;
967 off_t delta_offset, delta_data_offset, base_offset;
968 struct got_object_id base_id;
970 if (sigint_received)
971 return got_error(GOT_ERR_CANCELLED);
973 obj_idx = got_packidx_get_object_idx(a->packidx, id);
974 if (obj_idx == -1)
975 return NULL; /* object not present in our pack file */
977 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
978 &delta_compressed_size, &delta_offset, &delta_data_offset,
979 &base_offset, &base_id, &base_size, &result_size,
980 a->pack, a->packidx, obj_idx);
981 if (err) {
982 if (err->code == GOT_ERR_OBJ_TYPE)
983 return NULL; /* object not stored as a delta */
984 return err;
987 /*
988 * If this is an offset delta we must determine the base
989 * object ID ourselves.
990 */
991 if (base_offset != 0) {
992 err = get_base_object_id(&base_id, a->packidx, base_offset);
993 if (err)
994 goto done;
997 if (got_object_idset_contains(a->idset, &base_id)) {
998 struct got_imsg_reused_delta *delta;
1000 delta = &a->deltas[a->ndeltas++];
1001 memcpy(&delta->id, id, sizeof(delta->id));
1002 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
1003 delta->base_size = base_size;
1004 delta->result_size = result_size;
1005 delta->delta_size = delta_size;
1006 delta->delta_compressed_size = delta_compressed_size;
1007 delta->delta_offset = delta_data_offset;
1009 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
1010 err = got_privsep_send_reused_deltas(a->ibuf,
1011 a->deltas, a->ndeltas);
1012 if (err)
1013 goto done;
1014 a->ndeltas = 0;
1017 done:
1018 free(delta_buf);
1019 return err;
1022 static const struct got_error *
1023 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1025 const struct got_error *err = NULL;
1026 int done = 0;
1027 struct got_object_id *ids;
1028 size_t nids, i;
1030 for (;;) {
1031 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1032 if (err || done)
1033 break;
1034 for (i = 0; i < nids; i++) {
1035 err = got_object_idset_add(idset, &ids[i], NULL);
1036 if (err) {
1037 free(ids);
1038 return err;
1041 free(ids);
1044 return err;
1047 static const struct got_error *
1048 recv_object_id_queue(struct got_object_id_queue *queue,
1049 struct got_object_idset *queued_ids, struct imsgbuf *ibuf)
1051 const struct got_error *err = NULL;
1052 int done = 0;
1053 struct got_object_qid *qid;
1054 struct got_object_id *ids;
1055 size_t nids, i;
1057 for (;;) {
1058 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1059 if (err || done)
1060 break;
1061 for (i = 0; i < nids; i++) {
1062 err = got_object_qid_alloc_partial(&qid);
1063 if (err)
1064 return err;
1065 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1066 STAILQ_INSERT_TAIL(queue, qid, entry);
1067 err = got_object_idset_add(queued_ids, &qid->id, NULL);
1068 if (err)
1069 return err;
1073 return err;
1076 static const struct got_error *
1077 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1078 struct got_pack *pack, struct got_packidx *packidx)
1080 const struct got_error *err = NULL;
1081 struct got_object_idset *idset;
1082 struct search_deltas_arg sda;
1084 idset = got_object_idset_alloc();
1085 if (idset == NULL)
1086 return got_error_from_errno("got_object_idset_alloc");
1088 err = recv_object_ids(idset, ibuf);
1089 if (err)
1090 return err;
1092 memset(&sda, 0, sizeof(sda));
1093 sda.ibuf = ibuf;
1094 sda.idset = idset;
1095 sda.pack = pack;
1096 sda.packidx = packidx;
1097 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1098 if (err)
1099 goto done;
1101 if (sda.ndeltas > 0) {
1102 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1103 sda.ndeltas);
1104 if (err)
1105 goto done;
1108 err = got_privsep_send_reused_deltas_done(ibuf);
1109 done:
1110 got_object_idset_free(idset);
1111 return err;
1114 static const struct got_error *
1115 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1117 const struct got_error *err = NULL;
1118 struct imsg imsg;
1119 struct got_imsg_packidx ipackidx;
1120 size_t datalen;
1121 struct got_packidx *p;
1123 *packidx = NULL;
1125 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1126 if (err)
1127 return err;
1129 p = calloc(1, sizeof(*p));
1130 if (p == NULL) {
1131 err = got_error_from_errno("calloc");
1132 goto done;
1135 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1136 err = got_error(GOT_ERR_PRIVSEP_MSG);
1137 goto done;
1140 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1141 if (datalen != sizeof(ipackidx)) {
1142 err = got_error(GOT_ERR_PRIVSEP_LEN);
1143 goto done;
1145 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1147 p->fd = imsg_get_fd(&imsg);
1148 p->len = ipackidx.len;
1149 if (p->fd == -1) {
1150 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1151 goto done;
1153 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1154 err = got_error_from_errno("lseek");
1155 goto done;
1158 #ifndef GOT_PACK_NO_MMAP
1159 if (p->len > 0 && p->len <= SIZE_MAX) {
1160 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1161 if (p->map == MAP_FAILED)
1162 p->map = NULL; /* fall back to read(2) */
1164 #endif
1165 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1166 done:
1167 if (err) {
1168 if (p != NULL)
1169 got_packidx_close(p);
1170 } else
1171 *packidx = p;
1172 imsg_free(&imsg);
1173 return err;
1176 static const struct got_error *
1177 send_tree_enumeration_done(struct imsgbuf *ibuf)
1179 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1180 NULL, 0) == -1)
1181 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1183 return got_privsep_flush_imsg(ibuf);
1186 struct enumerated_tree {
1187 struct got_object_id id;
1188 char *path;
1189 uint8_t *buf;
1190 struct got_parsed_tree_entry *entries;
1191 int nentries;
1194 static const struct got_error *
1195 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1196 struct got_object_id *tree_id,
1197 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1198 struct got_object_cache *objcache, struct got_object_idset *idset,
1199 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1201 const struct got_error *err = NULL;
1202 struct got_object_id_queue ids;
1203 struct got_object_qid *qid;
1204 uint8_t *buf = NULL;
1205 size_t len = 0;
1206 struct got_parsed_tree_entry *entries = NULL;
1207 size_t nentries = 0, nentries_alloc = 0, i;
1208 struct enumerated_tree *tree;
1210 *ntrees = 0;
1211 *have_all_entries = 1;
1212 STAILQ_INIT(&ids);
1214 err = got_object_qid_alloc_partial(&qid);
1215 if (err)
1216 return err;
1217 memcpy(&qid->id, tree_id, sizeof(*tree_id));
1218 qid->data = strdup(path);
1219 if (qid->data == NULL) {
1220 err = got_error_from_errno("strdup");
1221 goto done;
1223 STAILQ_INSERT_TAIL(&ids, qid, entry);
1224 qid = NULL;
1226 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1227 do {
1228 const char *path;
1229 int idx, i;
1231 if (sigint_received) {
1232 err = got_error(GOT_ERR_CANCELLED);
1233 goto done;
1236 qid = STAILQ_FIRST(&ids);
1237 STAILQ_REMOVE_HEAD(&ids, entry);
1238 path = qid->data;
1240 idx = got_packidx_get_object_idx(packidx, &qid->id);
1241 if (idx == -1) {
1242 *have_all_entries = 0;
1243 break;
1246 err = open_tree(&buf, &len, pack, packidx, idx, &qid->id,
1247 objcache);
1248 if (err) {
1249 if (err->code != GOT_ERR_NO_OBJ)
1250 goto done;
1253 err = got_object_parse_tree(&entries, &nentries,
1254 &nentries_alloc, buf, len);
1255 if (err)
1256 goto done;
1258 err = got_object_idset_add(idset, &qid->id, NULL);
1259 if (err)
1260 goto done;
1262 for (i = 0; i < nentries; i++) {
1263 struct got_object_qid *eqid = NULL;
1264 struct got_parsed_tree_entry *pte = &entries[i];
1265 char *p;
1267 if (!S_ISDIR(pte->mode))
1268 continue;
1270 err = got_object_qid_alloc_partial(&eqid);
1271 if (err)
1272 goto done;
1273 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1275 if (got_object_idset_contains(idset, &eqid->id)) {
1276 got_object_qid_free(eqid);
1277 continue;
1280 if (asprintf(&p, "%s%s%s", path,
1281 got_path_is_root_dir(path) ? "" : "/",
1282 pte->name) == -1) {
1283 err = got_error_from_errno("asprintf");
1284 got_object_qid_free(eqid);
1285 goto done;
1287 eqid->data = p;
1288 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1291 if (*ntrees >= *nalloc) {
1292 struct enumerated_tree *new;
1293 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1294 sizeof(*new));
1295 if (new == NULL) {
1296 err = got_error_from_errno("malloc");
1297 goto done;
1299 *trees = new;
1300 *nalloc += 16;
1302 tree = &(*trees)[*ntrees];
1303 (*ntrees)++;
1304 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1305 tree->path = qid->data;
1306 tree->buf = buf;
1307 buf = NULL;
1308 tree->entries = entries;
1309 entries = NULL;
1310 nentries_alloc = 0;
1311 tree->nentries = nentries;
1312 nentries = 0;
1314 got_object_qid_free(qid);
1315 qid = NULL;
1316 } while (!STAILQ_EMPTY(&ids));
1318 if (*have_all_entries) {
1319 int i;
1321 * We have managed to traverse all entries in the hierarchy.
1322 * Tell the main process what we have found.
1324 for (i = 0; i < *ntrees; i++) {
1325 tree = &(*trees)[i];
1326 err = got_privsep_send_enumerated_tree(totlen,
1327 ibuf, &tree->id, tree->path, tree->entries,
1328 tree->nentries);
1329 if (err)
1330 goto done;
1331 free(tree->buf);
1332 tree->buf = NULL;
1333 free(tree->path);
1334 tree->path = NULL;
1335 free(tree->entries);
1336 tree->entries = NULL;
1338 *ntrees = 0; /* don't loop again below to free memory */
1340 err = send_tree_enumeration_done(ibuf);
1341 } else {
1343 * We can only load fully packed tree hierarchies on
1344 * behalf of the main process, otherwise the main process
1345 * gets a wrong idea about which tree objects have
1346 * already been traversed.
1347 * Indicate a missing entry for the root of this tree.
1348 * The main process should continue by loading this
1349 * entire tree the slow way.
1351 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1352 tree_id, "/", NULL, -1);
1353 if (err)
1354 goto done;
1356 done:
1357 free(buf);
1358 free(entries);
1359 for (i = 0; i < *ntrees; i++) {
1360 tree = &(*trees)[i];
1361 free(tree->buf);
1362 tree->buf = NULL;
1363 free(tree->path);
1364 tree->path = NULL;
1365 free(tree->entries);
1366 tree->entries = NULL;
1368 if (qid)
1369 free(qid->data);
1370 got_object_qid_free(qid);
1371 got_object_id_queue_free(&ids);
1372 if (err) {
1373 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1374 err = NULL;
1375 else
1376 got_privsep_send_error(ibuf, err);
1379 return err;
1382 static const struct got_error *
1383 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1384 struct got_pack *pack, struct got_packidx *packidx,
1385 struct got_object_cache *objcache)
1387 const struct got_error *err = NULL;
1388 struct got_object_id_queue commit_ids;
1389 const struct got_object_id_queue *parents = NULL;
1390 struct got_object_qid *qid = NULL;
1391 struct got_object *obj = NULL;
1392 struct got_commit_object *commit = NULL;
1393 struct got_object_id *tree_id = NULL;
1394 size_t totlen = 0;
1395 struct got_object_idset *idset, *queued_ids = NULL;
1396 int i, idx, have_all_entries = 1;
1397 struct enumerated_tree *trees = NULL;
1398 size_t ntrees = 0, nalloc = 16;
1400 STAILQ_INIT(&commit_ids);
1402 trees = calloc(nalloc, sizeof(*trees));
1403 if (trees == NULL)
1404 return got_error_from_errno("calloc");
1406 idset = got_object_idset_alloc();
1407 if (idset == NULL) {
1408 err = got_error_from_errno("got_object_idset_alloc");
1409 goto done;
1412 queued_ids = got_object_idset_alloc();
1413 if (queued_ids == NULL) {
1414 err = got_error_from_errno("got_object_idset_alloc");
1415 goto done;
1418 err = recv_object_id_queue(&commit_ids, queued_ids, ibuf);
1419 if (err)
1420 goto done;
1422 if (STAILQ_EMPTY(&commit_ids)) {
1423 err = got_error(GOT_ERR_PRIVSEP_MSG);
1424 goto done;
1427 err = recv_object_ids(idset, ibuf);
1428 if (err)
1429 goto done;
1431 while (!STAILQ_EMPTY(&commit_ids)) {
1432 if (sigint_received) {
1433 err = got_error(GOT_ERR_CANCELLED);
1434 goto done;
1437 qid = STAILQ_FIRST(&commit_ids);
1438 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1440 if (got_object_idset_contains(idset, &qid->id)) {
1441 got_object_qid_free(qid);
1442 qid = NULL;
1443 continue;
1446 idx = got_packidx_get_object_idx(packidx, &qid->id);
1447 if (idx == -1) {
1448 have_all_entries = 0;
1449 break;
1452 err = open_object(&obj, pack, packidx, idx, &qid->id,
1453 objcache);
1454 if (err)
1455 goto done;
1456 if (obj->type == GOT_OBJ_TYPE_TAG) {
1457 struct got_tag_object *tag;
1458 uint8_t *buf;
1459 size_t len;
1460 err = got_packfile_extract_object_to_mem(&buf,
1461 &len, obj, pack);
1462 if (err)
1463 goto done;
1464 obj->size = len;
1465 err = got_object_parse_tag(&tag, buf, len);
1466 if (err) {
1467 free(buf);
1468 goto done;
1470 idx = got_packidx_get_object_idx(packidx, &tag->id);
1471 if (idx == -1) {
1472 have_all_entries = 0;
1473 break;
1475 err = open_commit(&commit, pack, packidx, idx,
1476 &tag->id, objcache);
1477 got_object_tag_close(tag);
1478 free(buf);
1479 if (err)
1480 goto done;
1481 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1482 err = open_commit(&commit, pack, packidx, idx,
1483 &qid->id, objcache);
1484 if (err)
1485 goto done;
1486 } else {
1487 err = got_error(GOT_ERR_OBJ_TYPE);
1488 goto done;
1490 got_object_close(obj);
1491 obj = NULL;
1493 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1494 got_object_commit_get_committer_time(commit));
1495 if (err)
1496 goto done;
1498 tree_id = got_object_commit_get_tree_id(commit);
1499 idx = got_packidx_get_object_idx(packidx, tree_id);
1500 if (idx == -1) {
1501 have_all_entries = 0;
1502 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1503 tree_id, "/", NULL, -1);
1504 if (err)
1505 goto done;
1506 break;
1509 if (got_object_idset_contains(idset, tree_id)) {
1510 got_object_qid_free(qid);
1511 qid = NULL;
1512 err = send_tree_enumeration_done(ibuf);
1513 if (err)
1514 goto done;
1515 continue;
1518 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1519 tree_id, "/", pack, packidx, objcache, idset,
1520 &trees, &nalloc, &ntrees);
1521 if (err)
1522 goto done;
1524 if (!have_all_entries)
1525 break;
1527 got_object_qid_free(qid);
1528 qid = NULL;
1530 parents = got_object_commit_get_parent_ids(commit);
1531 if (parents) {
1532 struct got_object_qid *pid;
1533 STAILQ_FOREACH(pid, parents, entry) {
1534 if (got_object_idset_contains(idset, &pid->id))
1535 continue;
1536 if (got_object_idset_contains(queued_ids, &pid->id))
1537 continue;
1538 err = got_object_qid_alloc_partial(&qid);
1539 if (err)
1540 goto done;
1541 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1542 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1543 qid = NULL;
1547 got_object_commit_close(commit);
1548 commit = NULL;
1551 if (have_all_entries) {
1552 err = got_privsep_send_object_enumeration_done(ibuf);
1553 if (err)
1554 goto done;
1555 } else {
1556 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1557 if (err)
1558 goto done;
1560 done:
1561 if (obj)
1562 got_object_close(obj);
1563 if (commit)
1564 got_object_commit_close(commit);
1565 got_object_qid_free(qid);
1566 got_object_id_queue_free(&commit_ids);
1567 if (idset)
1568 got_object_idset_free(idset);
1569 if (queued_ids)
1570 got_object_idset_free(queued_ids);
1571 for (i = 0; i < ntrees; i++) {
1572 struct enumerated_tree *tree = &trees[i];
1573 free(tree->buf);
1574 free(tree->path);
1575 free(tree->entries);
1577 free(trees);
1578 return err;
1581 enum findtwixt_color {
1582 COLOR_KEEP = 0,
1583 COLOR_DROP,
1584 COLOR_SKIP,
1585 COLOR_MAX,
1588 static const struct got_error *
1589 paint_commit(struct got_object_qid *qid, intptr_t color)
1591 if (color < 0 || color >= COLOR_MAX)
1592 return got_error(GOT_ERR_RANGE);
1594 qid->data = (void *)color;
1595 return NULL;
1598 static const struct got_error *
1599 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1600 intptr_t color)
1602 const struct got_error *err;
1603 struct got_object_qid *qid;
1605 err = got_object_qid_alloc_partial(&qid);
1606 if (err)
1607 return err;
1609 memcpy(&qid->id, id, sizeof(qid->id));
1610 STAILQ_INSERT_TAIL(ids, qid, entry);
1611 return paint_commit(qid, color);
1614 static const struct got_error *
1615 paint_commits(struct got_object_id_queue *ids, int *nids,
1616 struct got_object_idset *keep, struct got_object_idset *drop,
1617 struct got_object_idset *skip, struct got_pack *pack,
1618 struct got_packidx *packidx, struct imsgbuf *ibuf,
1619 struct got_object_cache *objcache)
1621 const struct got_error *err = NULL;
1622 struct got_commit_object *commit = NULL;
1623 struct got_object_id_queue painted;
1624 const struct got_object_id_queue *parents;
1625 struct got_object_qid *qid = NULL;
1626 int nqueued = *nids, nskip = 0, npainted = 0;
1628 STAILQ_INIT(&painted);
1630 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1631 int idx;
1632 intptr_t color;
1634 if (sigint_received) {
1635 err = got_error(GOT_ERR_CANCELLED);
1636 goto done;
1639 qid = STAILQ_FIRST(ids);
1640 idx = got_packidx_get_object_idx(packidx, &qid->id);
1641 if (idx == -1) {
1642 qid = NULL;
1643 break;
1646 STAILQ_REMOVE_HEAD(ids, entry);
1647 nqueued--;
1648 color = (intptr_t)qid->data;
1649 if (color == COLOR_SKIP)
1650 nskip--;
1652 if (got_object_idset_contains(skip, &qid->id)) {
1653 got_object_qid_free(qid);
1654 qid = NULL;
1655 continue;
1658 switch (color) {
1659 case COLOR_KEEP:
1660 if (got_object_idset_contains(keep, &qid->id)) {
1661 got_object_qid_free(qid);
1662 qid = NULL;
1663 continue;
1665 if (got_object_idset_contains(drop, &qid->id)) {
1666 err = paint_commit(qid, COLOR_SKIP);
1667 if (err)
1668 goto done;
1670 err = got_object_idset_add(keep, &qid->id, NULL);
1671 if (err)
1672 goto done;
1673 break;
1674 case COLOR_DROP:
1675 if (got_object_idset_contains(drop, &qid->id)) {
1676 got_object_qid_free(qid);
1677 qid = NULL;
1678 continue;
1680 if (got_object_idset_contains(keep, &qid->id)) {
1681 err = paint_commit(qid, COLOR_SKIP);
1682 if (err)
1683 goto done;
1685 err = got_object_idset_add(drop, &qid->id, NULL);
1686 if (err)
1687 goto done;
1688 break;
1689 case COLOR_SKIP:
1690 if (!got_object_idset_contains(skip, &qid->id)) {
1691 err = got_object_idset_add(skip, &qid->id,
1692 NULL);
1693 if (err)
1694 goto done;
1696 break;
1697 default:
1698 /* should not happen */
1699 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1700 "%s invalid commit color %"PRIdPTR, __func__,
1701 color);
1702 goto done;
1705 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1706 objcache);
1707 if (err)
1708 goto done;
1710 parents = got_object_commit_get_parent_ids(commit);
1711 if (parents) {
1712 struct got_object_qid *pid;
1713 color = (intptr_t)qid->data;
1714 STAILQ_FOREACH(pid, parents, entry) {
1715 err = queue_commit_id(ids, &pid->id, color);
1716 if (err)
1717 goto done;
1718 nqueued++;
1719 if (color == COLOR_SKIP)
1720 nskip++;
1724 got_object_commit_close(commit);
1725 commit = NULL;
1727 STAILQ_INSERT_TAIL(&painted, qid, entry);
1728 qid = NULL;
1729 npainted++;
1731 err = got_privsep_send_painted_commits(ibuf, &painted,
1732 &npainted, 1, 0);
1733 if (err)
1734 goto done;
1737 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1738 if (err)
1739 goto done;
1741 *nids = nqueued;
1742 done:
1743 if (commit)
1744 got_object_commit_close(commit);
1745 got_object_qid_free(qid);
1746 return err;
1749 static void
1750 commit_painting_free(struct got_object_idset **keep,
1751 struct got_object_idset **drop,
1752 struct got_object_idset **skip)
1754 if (*keep) {
1755 got_object_idset_free(*keep);
1756 *keep = NULL;
1758 if (*drop) {
1759 got_object_idset_free(*drop);
1760 *drop = NULL;
1762 if (*skip) {
1763 got_object_idset_free(*skip);
1764 *skip = NULL;
1768 static const struct got_error *
1769 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1770 struct got_object_idset **drop, struct got_object_idset **skip)
1772 const struct got_error *err = NULL;
1774 *keep = got_object_idset_alloc();
1775 if (*keep == NULL) {
1776 err = got_error_from_errno("got_object_idset_alloc");
1777 goto done;
1779 *drop = got_object_idset_alloc();
1780 if (*drop == NULL) {
1781 err = got_error_from_errno("got_object_idset_alloc");
1782 goto done;
1784 *skip = got_object_idset_alloc();
1785 if (*skip == NULL) {
1786 err = got_error_from_errno("got_object_idset_alloc");
1787 goto done;
1790 err = recv_object_ids(*keep, ibuf);
1791 if (err)
1792 goto done;
1793 err = recv_object_ids(*drop, ibuf);
1794 if (err)
1795 goto done;
1796 err = recv_object_ids(*skip, ibuf);
1797 if (err)
1798 goto done;
1800 done:
1801 if (err)
1802 commit_painting_free(keep, drop, skip);
1804 return err;
1807 static const struct got_error *
1808 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1809 struct got_pack *pack, struct got_packidx *packidx,
1810 struct got_object_cache *objcache, struct got_object_idset *keep,
1811 struct got_object_idset *drop, struct got_object_idset *skip)
1813 const struct got_error *err = NULL;
1814 struct got_imsg_commit_painting_request ireq;
1815 struct got_object_id id;
1816 size_t datalen;
1817 struct got_object_id_queue ids;
1818 int nids = 0;
1820 STAILQ_INIT(&ids);
1822 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1823 if (datalen != sizeof(ireq))
1824 return got_error(GOT_ERR_PRIVSEP_LEN);
1825 memcpy(&ireq, imsg->data, sizeof(ireq));
1826 memcpy(&id, &ireq.id, sizeof(id));
1828 err = queue_commit_id(&ids, &id, ireq.color);
1829 if (err)
1830 return err;
1831 nids = 1;
1833 err = paint_commits(&ids, &nids, keep, drop, skip,
1834 pack, packidx, ibuf, objcache);
1835 if (err)
1836 goto done;
1838 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1839 if (err)
1840 goto done;
1842 err = got_privsep_send_painting_commits_done(ibuf);
1843 done:
1844 got_object_id_queue_free(&ids);
1845 return err;
1848 static const struct got_error *
1849 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1851 const struct got_error *err = NULL;
1852 struct imsg imsg;
1853 struct got_imsg_pack ipack;
1854 size_t datalen;
1855 struct got_pack *pack;
1857 *packp = NULL;
1859 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1860 if (err)
1861 return err;
1863 pack = calloc(1, sizeof(*pack));
1864 if (pack == NULL) {
1865 err = got_error_from_errno("calloc");
1866 goto done;
1869 if (imsg.hdr.type != GOT_IMSG_PACK) {
1870 err = got_error(GOT_ERR_PRIVSEP_MSG);
1871 goto done;
1874 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1875 if (datalen != sizeof(ipack)) {
1876 err = got_error(GOT_ERR_PRIVSEP_LEN);
1877 goto done;
1879 memcpy(&ipack, imsg.data, sizeof(ipack));
1881 pack->filesize = ipack.filesize;
1882 pack->fd = imsg_get_fd(&imsg);
1883 if (pack->fd == -1) {
1884 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1885 goto done;
1887 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1888 err = got_error_from_errno("lseek");
1889 goto done;
1891 pack->path_packfile = strdup(ipack.path_packfile);
1892 if (pack->path_packfile == NULL) {
1893 err = got_error_from_errno("strdup");
1894 goto done;
1897 err = got_delta_cache_alloc(&pack->delta_cache);
1898 if (err)
1899 goto done;
1901 #ifndef GOT_PACK_NO_MMAP
1902 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1903 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1904 pack->fd, 0);
1905 if (pack->map == MAP_FAILED)
1906 pack->map = NULL; /* fall back to read(2) */
1908 #endif
1909 done:
1910 if (err) {
1911 if (pack != NULL)
1912 got_pack_close(pack);
1913 } else
1914 *packp = pack;
1915 imsg_free(&imsg);
1916 return err;
1919 int
1920 main(int argc, char *argv[])
1922 const struct got_error *err = NULL;
1923 struct imsgbuf ibuf;
1924 struct imsg imsg;
1925 struct got_packidx *packidx = NULL;
1926 struct got_pack *pack = NULL;
1927 struct got_object_cache objcache;
1928 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1929 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1930 struct got_parsed_tree_entry *entries = NULL;
1931 size_t nentries = 0, nentries_alloc = 0;
1933 //static int attached;
1934 //while (!attached) sleep(1);
1936 signal(SIGINT, catch_sigint);
1938 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1940 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1941 if (err) {
1942 err = got_error_from_errno("got_object_cache_init");
1943 got_privsep_send_error(&ibuf, err);
1944 return 1;
1947 #ifndef PROFILE
1948 /* revoke access to most system calls */
1949 if (pledge("stdio recvfd", NULL) == -1) {
1950 err = got_error_from_errno("pledge");
1951 got_privsep_send_error(&ibuf, err);
1952 return 1;
1955 /* revoke fs access */
1956 if (landlock_no_fs() == -1) {
1957 err = got_error_from_errno("landlock_no_fs");
1958 got_privsep_send_error(&ibuf, err);
1959 return 1;
1961 if (cap_enter() == -1) {
1962 err = got_error_from_errno("cap_enter");
1963 got_privsep_send_error(&ibuf, err);
1964 return 1;
1966 #endif
1968 err = receive_packidx(&packidx, &ibuf);
1969 if (err) {
1970 got_privsep_send_error(&ibuf, err);
1971 return 1;
1974 err = receive_pack(&pack, &ibuf);
1975 if (err) {
1976 got_privsep_send_error(&ibuf, err);
1977 return 1;
1980 for (;;) {
1981 if (sigint_received) {
1982 err = got_error(GOT_ERR_CANCELLED);
1983 break;
1986 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1987 if (err) {
1988 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1989 err = NULL;
1990 break;
1993 if (imsg.hdr.type == GOT_IMSG_STOP) {
1994 imsg_free(&imsg);
1995 break;
1998 switch (imsg.hdr.type) {
1999 case GOT_IMSG_TMPFD:
2000 if (basefile == NULL) {
2001 err = receive_tempfile(&basefile, "w+",
2002 &imsg, &ibuf);
2003 } else if (accumfile == NULL) {
2004 err = receive_tempfile(&accumfile, "w+",
2005 &imsg, &ibuf);
2006 } else
2007 err = got_error(GOT_ERR_PRIVSEP_MSG);
2008 break;
2009 case GOT_IMSG_PACKED_OBJECT_REQUEST:
2010 err = object_request(&imsg, &ibuf, pack, packidx,
2011 &objcache);
2012 break;
2013 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
2014 if (basefile == NULL || accumfile == NULL) {
2015 err = got_error(GOT_ERR_PRIVSEP_MSG);
2016 break;
2018 err = raw_object_request(&imsg, &ibuf, pack, packidx,
2019 &objcache, basefile, accumfile);
2020 break;
2021 case GOT_IMSG_RAW_DELTA_OUTFD:
2022 if (delta_outfile != NULL) {
2023 err = got_error(GOT_ERR_PRIVSEP_MSG);
2024 break;
2026 err = receive_tempfile(&delta_outfile, "w",
2027 &imsg, &ibuf);
2028 break;
2029 case GOT_IMSG_RAW_DELTA_REQUEST:
2030 if (delta_outfile == NULL) {
2031 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2032 break;
2034 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2035 pack, packidx);
2036 break;
2037 case GOT_IMSG_DELTA_REUSE_REQUEST:
2038 err = delta_reuse_request(&imsg, &ibuf, pack, packidx);
2039 break;
2040 case GOT_IMSG_COMMIT_REQUEST:
2041 err = commit_request(&imsg, &ibuf, pack, packidx,
2042 &objcache);
2043 break;
2044 case GOT_IMSG_TREE_REQUEST:
2045 err = tree_request(&imsg, &ibuf, pack, packidx,
2046 &objcache, &entries, &nentries, &nentries_alloc);
2047 break;
2048 case GOT_IMSG_BLOB_REQUEST:
2049 if (basefile == NULL || accumfile == NULL) {
2050 err = got_error(GOT_ERR_PRIVSEP_MSG);
2051 break;
2053 err = blob_request(&imsg, &ibuf, pack, packidx,
2054 &objcache, basefile, accumfile);
2055 break;
2056 case GOT_IMSG_TAG_REQUEST:
2057 err = tag_request(&imsg, &ibuf, pack, packidx,
2058 &objcache);
2059 break;
2060 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2061 err = commit_traversal_request(&imsg, &ibuf, pack,
2062 packidx, &objcache);
2063 break;
2064 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2065 err = enumeration_request(&imsg, &ibuf, pack,
2066 packidx, &objcache);
2067 break;
2068 case GOT_IMSG_COMMIT_PAINTING_INIT:
2069 commit_painting_free(&keep, &drop, &skip);
2070 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2071 break;
2072 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2073 if (keep == NULL || drop == NULL || skip == NULL) {
2074 err = got_error(GOT_ERR_PRIVSEP_MSG);
2075 break;
2077 err = commit_painting_request(&imsg, &ibuf, pack,
2078 packidx, &objcache, keep, drop, skip);
2079 break;
2080 case GOT_IMSG_COMMIT_PAINTING_DONE:
2081 commit_painting_free(&keep, &drop, &skip);
2082 break;
2083 default:
2084 err = got_error(GOT_ERR_PRIVSEP_MSG);
2085 break;
2088 imsg_free(&imsg);
2089 if (err)
2090 break;
2093 free(entries);
2094 commit_painting_free(&keep, &drop, &skip);
2095 if (packidx)
2096 got_packidx_close(packidx);
2097 if (pack)
2098 got_pack_close(pack);
2099 got_object_cache_close(&objcache);
2100 imsg_clear(&ibuf);
2101 if (basefile && fclose(basefile) == EOF && err == NULL)
2102 err = got_error_from_errno("fclose");
2103 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2104 err = got_error_from_errno("fclose");
2105 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2106 err = got_error_from_errno("fclose");
2107 if (err) {
2108 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2109 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2110 got_privsep_send_error(&ibuf, err);
2113 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2114 err = got_error_from_errno("close");
2115 return err ? 1 : 0;