Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/stat.h>
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
21 #include <sys/time.h>
22 #include <sys/mman.h>
24 #include <inttypes.h>
25 #include <limits.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <imsg.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <sha1.h>
33 #include <sha2.h>
34 #include <unistd.h>
35 #include <zlib.h>
37 #include "got_error.h"
38 #include "got_object.h"
39 #include "got_path.h"
41 #include "got_lib_delta.h"
42 #include "got_lib_delta_cache.h"
43 #include "got_lib_object.h"
44 #include "got_lib_object_qid.h"
45 #include "got_lib_object_cache.h"
46 #include "got_lib_object_parse.h"
47 #include "got_lib_object_idset.h"
48 #include "got_lib_privsep.h"
49 #include "got_lib_pack.h"
51 #ifndef nitems
52 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
53 #endif
55 static volatile sig_atomic_t sigint_received;
57 static void
58 catch_sigint(int signo)
59 {
60 sigint_received = 1;
61 }
63 static const struct got_error *
64 open_object(struct got_object **obj, struct got_pack *pack,
65 struct got_packidx *packidx, int idx, struct got_object_id *id,
66 struct got_object_cache *objcache)
67 {
68 const struct got_error *err;
70 err = got_packfile_open_object(obj, pack, packidx, idx, id);
71 if (err)
72 return err;
73 (*obj)->refcnt++;
75 err = got_object_cache_add(objcache, id, *obj);
76 if (err) {
77 if (err->code == GOT_ERR_OBJ_EXISTS ||
78 err->code == GOT_ERR_OBJ_TOO_LARGE)
79 err = NULL;
80 return err;
81 }
82 (*obj)->refcnt++;
83 return NULL;
84 }
86 static const struct got_error *
87 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
88 struct got_packidx *packidx, struct got_object_cache *objcache)
89 {
90 const struct got_error *err = NULL;
91 struct got_imsg_packed_object iobj;
92 struct got_object *obj;
93 struct got_object_id id;
94 size_t datalen;
96 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
97 if (datalen != sizeof(iobj))
98 return got_error(GOT_ERR_PRIVSEP_LEN);
99 memcpy(&iobj, imsg->data, sizeof(iobj));
100 memcpy(&id, &iobj.id, sizeof(id));
102 obj = got_object_cache_get(objcache, &id);
103 if (obj) {
104 obj->refcnt++;
105 } else {
106 err = open_object(&obj, pack, packidx, iobj.idx, &id,
107 objcache);
108 if (err)
109 goto done;
112 err = got_privsep_send_obj(ibuf, obj);
113 done:
114 got_object_close(obj);
115 return err;
118 static const struct got_error *
119 open_commit(struct got_commit_object **commit, struct got_pack *pack,
120 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
121 struct got_object_cache *objcache)
123 const struct got_error *err = NULL;
124 struct got_object *obj = NULL;
125 uint8_t *buf = NULL;
126 size_t len;
128 *commit = NULL;
130 obj = got_object_cache_get(objcache, id);
131 if (obj) {
132 obj->refcnt++;
133 } else {
134 err = open_object(&obj, pack, packidx, obj_idx, id,
135 objcache);
136 if (err)
137 return err;
140 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
141 if (err)
142 goto done;
144 obj->size = len;
146 err = got_object_parse_commit(commit, buf, len);
147 done:
148 got_object_close(obj);
149 free(buf);
150 return err;
153 static const struct got_error *
154 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
155 struct got_packidx *packidx, struct got_object_cache *objcache)
157 const struct got_error *err = NULL;
158 struct got_imsg_packed_object iobj;
159 struct got_commit_object *commit = NULL;
160 struct got_object_id id;
161 size_t datalen;
163 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
164 if (datalen != sizeof(iobj))
165 return got_error(GOT_ERR_PRIVSEP_LEN);
166 memcpy(&iobj, imsg->data, sizeof(iobj));
167 memcpy(&id, &iobj.id, sizeof(id));
169 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
170 if (err)
171 goto done;
173 err = got_privsep_send_commit(ibuf, commit);
174 done:
175 if (commit)
176 got_object_commit_close(commit);
177 if (err) {
178 if (err->code == GOT_ERR_PRIVSEP_PIPE)
179 err = NULL;
180 else
181 got_privsep_send_error(ibuf, err);
184 return err;
187 static const struct got_error *
188 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, size_t *nentries,
189 size_t *nentries_alloc, struct got_pack *pack, struct got_packidx *packidx,
190 int obj_idx, struct got_object_id *id, struct got_object_cache *objcache)
192 const struct got_error *err = NULL;
193 struct got_object *obj = NULL;
194 size_t len;
196 *buf = NULL;
197 *nentries = 0;
199 obj = got_object_cache_get(objcache, id);
200 if (obj) {
201 obj->refcnt++;
202 } else {
203 err = open_object(&obj, pack, packidx, obj_idx, id,
204 objcache);
205 if (err)
206 return err;
209 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
210 if (err)
211 goto done;
213 obj->size = len;
215 err = got_object_parse_tree(entries, nentries, nentries_alloc,
216 *buf, len);
217 done:
218 got_object_close(obj);
219 if (err) {
220 free(*buf);
221 *buf = NULL;
223 return err;
226 static const struct got_error *
227 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
228 struct got_packidx *packidx, struct got_object_cache *objcache,
229 struct got_parsed_tree_entry **entries, size_t *nentries,
230 size_t *nentries_alloc)
232 const struct got_error *err = NULL;
233 struct got_imsg_packed_object iobj;
234 uint8_t *buf = NULL;
235 struct got_object_id id;
236 size_t datalen;
238 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
239 if (datalen != sizeof(iobj))
240 return got_error(GOT_ERR_PRIVSEP_LEN);
241 memcpy(&iobj, imsg->data, sizeof(iobj));
242 memcpy(&id, &iobj.id, sizeof(id));
244 err = open_tree(&buf, entries, nentries, nentries_alloc,
245 pack, packidx, iobj.idx, &id, objcache);
246 if (err)
247 return err;
249 err = got_privsep_send_tree(ibuf, *entries, *nentries);
250 free(buf);
251 if (err) {
252 if (err->code == GOT_ERR_PRIVSEP_PIPE)
253 err = NULL;
254 else
255 got_privsep_send_error(ibuf, err);
258 return err;
261 static const struct got_error *
262 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
264 const struct got_error *err;
265 struct imsg imsg;
266 size_t datalen;
268 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
269 if (err)
270 return err;
272 if (imsg.hdr.type != imsg_code) {
273 err = got_error(GOT_ERR_PRIVSEP_MSG);
274 goto done;
277 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
278 if (datalen != 0) {
279 err = got_error(GOT_ERR_PRIVSEP_LEN);
280 goto done;
282 if (imsg.fd == -1) {
283 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
284 goto done;
287 *f = fdopen(imsg.fd, "w+");
288 if (*f == NULL) {
289 err = got_error_from_errno("fdopen");
290 close(imsg.fd);
291 goto done;
293 done:
294 imsg_free(&imsg);
295 return err;
298 static const struct got_error *
299 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
300 struct imsgbuf *ibuf)
302 size_t datalen;
304 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
305 if (datalen != 0)
306 return got_error(GOT_ERR_PRIVSEP_LEN);
308 if (imsg->fd == -1)
309 return got_error(GOT_ERR_PRIVSEP_NO_FD);
311 *f = fdopen(imsg->fd, mode);
312 if (*f == NULL)
313 return got_error_from_errno("fdopen");
314 imsg->fd = -1;
316 return NULL;
319 static const struct got_error *
320 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
321 struct got_packidx *packidx, struct got_object_cache *objcache,
322 FILE *basefile, FILE *accumfile)
324 const struct got_error *err = NULL;
325 struct got_imsg_packed_object iobj;
326 struct got_object *obj = NULL;
327 FILE *outfile = NULL;
328 struct got_object_id id;
329 size_t datalen;
330 uint64_t blob_size;
331 uint8_t *buf = NULL;
333 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
334 if (datalen != sizeof(iobj))
335 return got_error(GOT_ERR_PRIVSEP_LEN);
336 memcpy(&iobj, imsg->data, sizeof(iobj));
337 memcpy(&id, &iobj.id, sizeof(id));
339 obj = got_object_cache_get(objcache, &id);
340 if (obj) {
341 obj->refcnt++;
342 } else {
343 err = open_object(&obj, pack, packidx, iobj.idx, &id,
344 objcache);
345 if (err)
346 return err;
349 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
350 if (err)
351 goto done;
353 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
354 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
355 if (err)
356 goto done;
357 } else
358 blob_size = obj->size;
360 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
361 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
362 obj, pack);
363 else
364 err = got_packfile_extract_object(pack, obj, outfile, basefile,
365 accumfile);
366 if (err)
367 goto done;
369 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
370 done:
371 free(buf);
372 if (outfile && fclose(outfile) == EOF && err == NULL)
373 err = got_error_from_errno("fclose");
374 got_object_close(obj);
375 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
376 got_privsep_send_error(ibuf, err);
378 return err;
381 static const struct got_error *
382 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
383 struct got_packidx *packidx, struct got_object_cache *objcache)
385 const struct got_error *err = NULL;
386 struct got_imsg_packed_object iobj;
387 struct got_object *obj = NULL;
388 struct got_tag_object *tag = NULL;
389 uint8_t *buf = NULL;
390 size_t len;
391 struct got_object_id id;
392 size_t datalen;
394 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
395 if (datalen != sizeof(iobj))
396 return got_error(GOT_ERR_PRIVSEP_LEN);
397 memcpy(&iobj, imsg->data, sizeof(iobj));
398 memcpy(&id, &iobj.id, sizeof(id));
400 obj = got_object_cache_get(objcache, &id);
401 if (obj) {
402 obj->refcnt++;
403 } else {
404 err = open_object(&obj, pack, packidx, iobj.idx, &id,
405 objcache);
406 if (err)
407 return err;
410 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
411 if (err)
412 goto done;
414 obj->size = len;
415 err = got_object_parse_tag(&tag, buf, len);
416 if (err)
417 goto done;
419 err = got_privsep_send_tag(ibuf, tag);
420 done:
421 free(buf);
422 got_object_close(obj);
423 if (tag)
424 got_object_tag_close(tag);
425 if (err) {
426 if (err->code == GOT_ERR_PRIVSEP_PIPE)
427 err = NULL;
428 else
429 got_privsep_send_error(ibuf, err);
432 return err;
435 static struct got_parsed_tree_entry *
436 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
437 const char *name, size_t len)
439 struct got_parsed_tree_entry *pte;
440 int cmp, i;
442 /* Note that tree entries are sorted in strncmp() order. */
443 for (i = 0; i < nentries; i++) {
444 pte = &entries[i];
445 cmp = strncmp(pte->name, name, len);
446 if (cmp < 0)
447 continue;
448 if (cmp > 0)
449 break;
450 if (pte->name[len] == '\0')
451 return pte;
453 return NULL;
456 static const struct got_error *
457 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
458 struct got_parsed_tree_entry **entries1, size_t *nentries1,
459 size_t *nentries_alloc1,
460 struct got_parsed_tree_entry **entries2, size_t *nentries2,
461 size_t *nentries_alloc2,
462 const char *path, struct got_pack *pack, struct got_packidx *packidx,
463 struct imsgbuf *ibuf, struct got_object_cache *objcache)
465 const struct got_error *err = NULL;
466 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
467 const char *seg, *s;
468 size_t seglen;
470 *changed = 0;
472 /* We not do support comparing the root path. */
473 if (got_path_is_root_dir(path))
474 return got_error_path(path, GOT_ERR_BAD_PATH);
476 s = path;
477 while (*s == '/')
478 s++;
479 seg = s;
480 seglen = 0;
481 while (*s) {
482 if (*s != '/') {
483 s++;
484 seglen++;
485 if (*s)
486 continue;
489 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
490 if (pte1 == NULL) {
491 err = got_error(GOT_ERR_NO_OBJ);
492 break;
495 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
496 if (pte2 == NULL) {
497 *changed = 1;
498 break;
501 if (pte1->mode != pte2->mode) {
502 *changed = 1;
503 break;
506 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
507 *changed = 0;
508 break;
511 if (*s == '\0') { /* final path element */
512 *changed = 1;
513 break;
516 seg = s + 1;
517 s++;
518 seglen = 0;
519 if (*s) {
520 struct got_object_id id1, id2;
521 int idx;
523 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
524 idx = got_packidx_get_object_idx(packidx, &id1);
525 if (idx == -1) {
526 err = got_error_no_obj(&id1);
527 break;
529 *nentries1 = 0;
530 free(*buf1);
531 *buf1 = NULL;
532 err = open_tree(buf1, entries1, nentries1,
533 nentries_alloc1, pack, packidx, idx, &id1,
534 objcache);
535 pte1 = NULL;
536 if (err)
537 break;
539 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
540 idx = got_packidx_get_object_idx(packidx, &id2);
541 if (idx == -1) {
542 err = got_error_no_obj(&id2);
543 break;
545 *nentries2 = 0;
546 free(*buf2);
547 *buf2 = NULL;
548 err = open_tree(buf2, entries2, nentries2,
549 nentries_alloc2, pack, packidx, idx, &id2,
550 objcache);
551 pte2 = NULL;
552 if (err)
553 break;
557 return err;
560 static const struct got_error *
561 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
562 struct imsgbuf *ibuf)
564 struct ibuf *wbuf;
565 size_t i;
567 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
568 sizeof(struct got_imsg_traversed_commits) +
569 ncommits * sizeof(commit_ids[0]));
570 if (wbuf == NULL)
571 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
573 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
574 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
576 for (i = 0; i < ncommits; i++) {
577 struct got_object_id *id = &commit_ids[i];
578 if (imsg_add(wbuf, id, sizeof(*id)) == -1) {
579 return got_error_from_errno(
580 "imsg_add TRAVERSED_COMMITS");
584 wbuf->fd = -1;
585 imsg_close(ibuf, wbuf);
587 return got_privsep_flush_imsg(ibuf);
590 static const struct got_error *
591 send_commit_traversal_done(struct imsgbuf *ibuf)
593 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
594 NULL, 0) == -1)
595 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
597 return got_privsep_flush_imsg(ibuf);
600 static const struct got_error *
601 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
602 struct got_pack *pack, struct got_packidx *packidx,
603 struct got_object_cache *objcache)
605 const struct got_error *err = NULL;
606 struct got_imsg_commit_traversal_request ctreq;
607 struct got_object_qid *pid;
608 struct got_commit_object *commit = NULL, *pcommit = NULL;
609 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
610 size_t nentries = 0, nentries_alloc = 0;
611 size_t pnentries = 0, pnentries_alloc = 0;
612 struct got_object_id id;
613 size_t datalen;
614 char *path = NULL;
615 const int min_alloc = 64;
616 int changed = 0, ncommits = 0, nallocated = 0;
617 struct got_object_id *commit_ids = NULL;
619 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
620 if (datalen < sizeof(ctreq))
621 return got_error(GOT_ERR_PRIVSEP_LEN);
622 memcpy(&ctreq, imsg->data, sizeof(ctreq));
623 memcpy(&id, &ctreq.iobj.id, sizeof(id));
625 if (datalen != sizeof(ctreq) + ctreq.path_len)
626 return got_error(GOT_ERR_PRIVSEP_LEN);
627 if (ctreq.path_len == 0)
628 return got_error(GOT_ERR_PRIVSEP_LEN);
630 path = strndup(imsg->data + sizeof(ctreq), ctreq.path_len);
631 if (path == NULL)
632 return got_error_from_errno("strndup");
634 nallocated = min_alloc;
635 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
636 if (commit_ids == NULL)
637 return got_error_from_errno("reallocarray");
639 do {
640 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
641 int idx;
643 if (sigint_received) {
644 err = got_error(GOT_ERR_CANCELLED);
645 goto done;
648 if (commit == NULL) {
649 idx = got_packidx_get_object_idx(packidx, &id);
650 if (idx == -1)
651 break;
652 err = open_commit(&commit, pack, packidx,
653 idx, &id, objcache);
654 if (err) {
655 if (err->code != GOT_ERR_NO_OBJ)
656 goto done;
657 err = NULL;
658 break;
662 if (sizeof(struct got_imsg_traversed_commits) +
663 ncommits * sizeof(commit_ids[0]) >= max_datalen) {
664 err = send_traversed_commits(commit_ids, ncommits,
665 ibuf);
666 if (err)
667 goto done;
668 ncommits = 0;
670 ncommits++;
671 if (ncommits > nallocated) {
672 struct got_object_id *new;
673 nallocated += min_alloc;
674 new = reallocarray(commit_ids, nallocated,
675 sizeof(*commit_ids));
676 if (new == NULL) {
677 err = got_error_from_errno("reallocarray");
678 goto done;
680 commit_ids = new;
682 memcpy(&commit_ids[ncommits - 1], &id, sizeof(id));
684 pid = STAILQ_FIRST(&commit->parent_ids);
685 if (pid == NULL)
686 break;
688 idx = got_packidx_get_object_idx(packidx, &pid->id);
689 if (idx == -1)
690 break;
692 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
693 objcache);
694 if (err) {
695 if (err->code != GOT_ERR_NO_OBJ)
696 goto done;
697 err = NULL;
698 break;
701 if (path[0] == '/' && path[1] == '\0') {
702 if (got_object_id_cmp(pcommit->tree_id,
703 commit->tree_id) != 0) {
704 changed = 1;
705 break;
707 } else {
708 int pidx;
709 uint8_t *buf = NULL, *pbuf = NULL;
711 idx = got_packidx_get_object_idx(packidx,
712 commit->tree_id);
713 if (idx == -1)
714 break;
715 pidx = got_packidx_get_object_idx(packidx,
716 pcommit->tree_id);
717 if (pidx == -1)
718 break;
720 err = open_tree(&buf, &entries, &nentries,
721 &nentries_alloc, pack, packidx, idx,
722 commit->tree_id, objcache);
723 if (err)
724 goto done;
725 err = open_tree(&pbuf, &pentries, &pnentries,
726 &pnentries_alloc, pack, packidx, pidx,
727 pcommit->tree_id, objcache);
728 if (err) {
729 free(buf);
730 goto done;
733 err = tree_path_changed(&changed, &buf, &pbuf,
734 &entries, &nentries, &nentries_alloc,
735 &pentries, &pnentries, &pnentries_alloc,
736 path, pack, packidx, ibuf, objcache);
738 nentries = 0;
739 free(buf);
740 pnentries = 0;
741 free(pbuf);
742 if (err) {
743 if (err->code != GOT_ERR_NO_OBJ)
744 goto done;
745 err = NULL;
746 break;
750 if (!changed) {
751 memcpy(&id, &pid->id, sizeof(id));
752 got_object_commit_close(commit);
753 commit = pcommit;
754 pcommit = NULL;
756 } while (!changed);
758 if (ncommits > 0) {
759 err = send_traversed_commits(commit_ids, ncommits, ibuf);
760 if (err)
761 goto done;
763 if (changed) {
764 err = got_privsep_send_commit(ibuf, commit);
765 if (err)
766 goto done;
769 err = send_commit_traversal_done(ibuf);
770 done:
771 free(path);
772 free(commit_ids);
773 if (commit)
774 got_object_commit_close(commit);
775 if (pcommit)
776 got_object_commit_close(pcommit);
777 free(entries);
778 free(pentries);
779 if (err) {
780 if (err->code == GOT_ERR_PRIVSEP_PIPE)
781 err = NULL;
782 else
783 got_privsep_send_error(ibuf, err);
786 return err;
789 static const struct got_error *
790 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
791 struct got_pack *pack, struct got_packidx *packidx,
792 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
794 const struct got_error *err = NULL;
795 uint8_t *buf = NULL;
796 uint64_t size = 0;
797 FILE *outfile = NULL;
798 struct got_imsg_packed_object iobj;
799 struct got_object *obj;
800 struct got_object_id id;
801 size_t datalen;
803 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
804 if (datalen != sizeof(iobj))
805 return got_error(GOT_ERR_PRIVSEP_LEN);
806 memcpy(&iobj, imsg->data, sizeof(iobj));
807 memcpy(&id, &iobj.id, sizeof(id));
809 obj = got_object_cache_get(objcache, &id);
810 if (obj) {
811 obj->refcnt++;
812 } else {
813 err = open_object(&obj, pack, packidx, iobj.idx, &id,
814 objcache);
815 if (err)
816 return err;
819 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
820 if (err)
821 return err;
823 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
824 err = got_pack_get_max_delta_object_size(&size, obj, pack);
825 if (err)
826 goto done;
827 } else
828 size = obj->size;
830 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
831 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
832 obj, pack);
833 else
834 err = got_packfile_extract_object(pack, obj, outfile, basefile,
835 accumfile);
836 if (err)
837 goto done;
839 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
840 done:
841 free(buf);
842 if (outfile && fclose(outfile) == EOF && err == NULL)
843 err = got_error_from_errno("fclose");
844 got_object_close(obj);
845 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
846 got_privsep_send_error(ibuf, err);
848 return err;
851 static const struct got_error *
852 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
853 off_t base_offset)
855 const struct got_error *err;
856 int idx;
858 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
859 if (err)
860 return err;
861 if (idx == -1)
862 return got_error(GOT_ERR_BAD_PACKIDX);
864 return got_packidx_get_object_id(base_id, packidx, idx);
867 static const struct got_error *
868 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
869 FILE *delta_outfile, struct got_pack *pack,
870 struct got_packidx *packidx)
872 const struct got_error *err = NULL;
873 struct got_imsg_raw_delta_request req;
874 size_t datalen, delta_size, delta_compressed_size;
875 off_t delta_offset, delta_data_offset;
876 uint8_t *delta_buf = NULL;
877 struct got_object_id id, base_id;
878 off_t base_offset, delta_out_offset = 0;
879 uint64_t base_size = 0, result_size = 0;
880 size_t w;
882 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
883 if (datalen != sizeof(req))
884 return got_error(GOT_ERR_PRIVSEP_LEN);
885 memcpy(&req, imsg->data, sizeof(req));
886 memcpy(&id, &req.id, sizeof(id));
888 imsg->fd = -1;
890 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
891 &delta_compressed_size, &delta_offset, &delta_data_offset,
892 &base_offset, &base_id, &base_size, &result_size,
893 pack, packidx, req.idx);
894 if (err)
895 goto done;
897 /*
898 * If this is an offset delta we must determine the base
899 * object ID ourselves.
900 */
901 if (base_offset != 0) {
902 err = get_base_object_id(&base_id, packidx, base_offset);
903 if (err)
904 goto done;
907 delta_out_offset = ftello(delta_outfile);
908 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
909 if (w != delta_compressed_size) {
910 err = got_ferror(delta_outfile, GOT_ERR_IO);
911 goto done;
913 if (fflush(delta_outfile) == -1) {
914 err = got_error_from_errno("fflush");
915 goto done;
918 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
919 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
920 &base_id);
921 done:
922 free(delta_buf);
923 return err;
926 struct search_deltas_arg {
927 struct imsgbuf *ibuf;
928 struct got_packidx *packidx;
929 struct got_pack *pack;
930 struct got_object_idset *idset;
931 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
932 size_t ndeltas;
933 };
935 static const struct got_error *
936 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
938 const struct got_error *err;
939 struct search_deltas_arg *a = arg;
940 int obj_idx;
941 uint8_t *delta_buf = NULL;
942 uint64_t base_size, result_size;
943 size_t delta_size, delta_compressed_size;
944 off_t delta_offset, delta_data_offset, base_offset;
945 struct got_object_id base_id;
947 if (sigint_received)
948 return got_error(GOT_ERR_CANCELLED);
950 obj_idx = got_packidx_get_object_idx(a->packidx, id);
951 if (obj_idx == -1)
952 return NULL; /* object not present in our pack file */
954 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
955 &delta_compressed_size, &delta_offset, &delta_data_offset,
956 &base_offset, &base_id, &base_size, &result_size,
957 a->pack, a->packidx, obj_idx);
958 if (err) {
959 if (err->code == GOT_ERR_OBJ_TYPE)
960 return NULL; /* object not stored as a delta */
961 return err;
964 /*
965 * If this is an offset delta we must determine the base
966 * object ID ourselves.
967 */
968 if (base_offset != 0) {
969 err = get_base_object_id(&base_id, a->packidx, base_offset);
970 if (err)
971 goto done;
974 if (got_object_idset_contains(a->idset, &base_id)) {
975 struct got_imsg_reused_delta *delta;
977 delta = &a->deltas[a->ndeltas++];
978 memcpy(&delta->id, id, sizeof(delta->id));
979 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
980 delta->base_size = base_size;
981 delta->result_size = result_size;
982 delta->delta_size = delta_size;
983 delta->delta_compressed_size = delta_compressed_size;
984 delta->delta_offset = delta_data_offset;
986 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
987 err = got_privsep_send_reused_deltas(a->ibuf,
988 a->deltas, a->ndeltas);
989 if (err)
990 goto done;
991 a->ndeltas = 0;
994 done:
995 free(delta_buf);
996 return err;
999 static const struct got_error *
1000 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1002 const struct got_error *err = NULL;
1003 int done = 0;
1004 struct got_object_id *ids;
1005 size_t nids, i;
1007 for (;;) {
1008 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1009 if (err || done)
1010 break;
1011 for (i = 0; i < nids; i++) {
1012 err = got_object_idset_add(idset, &ids[i], NULL);
1013 if (err) {
1014 free(ids);
1015 return err;
1018 free(ids);
1021 return err;
1024 static const struct got_error *
1025 recv_object_id_queue(struct got_object_id_queue *queue,
1026 struct got_object_idset *queued_ids, struct imsgbuf *ibuf)
1028 const struct got_error *err = NULL;
1029 int done = 0;
1030 struct got_object_qid *qid;
1031 struct got_object_id *ids;
1032 size_t nids, i;
1034 for (;;) {
1035 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1036 if (err || done)
1037 break;
1038 for (i = 0; i < nids; i++) {
1039 err = got_object_qid_alloc_partial(&qid);
1040 if (err)
1041 return err;
1042 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1043 STAILQ_INSERT_TAIL(queue, qid, entry);
1044 err = got_object_idset_add(queued_ids, &qid->id, NULL);
1045 if (err)
1046 return err;
1050 return err;
1053 static const struct got_error *
1054 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1055 struct got_pack *pack, struct got_packidx *packidx)
1057 const struct got_error *err = NULL;
1058 struct got_object_idset *idset;
1059 struct search_deltas_arg sda;
1061 idset = got_object_idset_alloc();
1062 if (idset == NULL)
1063 return got_error_from_errno("got_object_idset_alloc");
1065 err = recv_object_ids(idset, ibuf);
1066 if (err)
1067 return err;
1069 memset(&sda, 0, sizeof(sda));
1070 sda.ibuf = ibuf;
1071 sda.idset = idset;
1072 sda.pack = pack;
1073 sda.packidx = packidx;
1074 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1075 if (err)
1076 goto done;
1078 if (sda.ndeltas > 0) {
1079 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1080 sda.ndeltas);
1081 if (err)
1082 goto done;
1085 err = got_privsep_send_reused_deltas_done(ibuf);
1086 done:
1087 got_object_idset_free(idset);
1088 return err;
1091 static const struct got_error *
1092 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1094 const struct got_error *err = NULL;
1095 struct imsg imsg;
1096 struct got_imsg_packidx ipackidx;
1097 size_t datalen;
1098 struct got_packidx *p;
1100 *packidx = NULL;
1102 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1103 if (err)
1104 return err;
1106 p = calloc(1, sizeof(*p));
1107 if (p == NULL) {
1108 err = got_error_from_errno("calloc");
1109 goto done;
1112 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1113 err = got_error(GOT_ERR_PRIVSEP_MSG);
1114 goto done;
1117 if (imsg.fd == -1) {
1118 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1119 goto done;
1122 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1123 if (datalen != sizeof(ipackidx)) {
1124 err = got_error(GOT_ERR_PRIVSEP_LEN);
1125 goto done;
1127 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1129 p->len = ipackidx.len;
1130 p->fd = dup(imsg.fd);
1131 if (p->fd == -1) {
1132 err = got_error_from_errno("dup");
1133 goto done;
1135 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1136 err = got_error_from_errno("lseek");
1137 goto done;
1140 #ifndef GOT_PACK_NO_MMAP
1141 if (p->len > 0 && p->len <= SIZE_MAX) {
1142 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1143 if (p->map == MAP_FAILED)
1144 p->map = NULL; /* fall back to read(2) */
1146 #endif
1147 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1148 done:
1149 if (err) {
1150 if (imsg.fd != -1)
1151 close(imsg.fd);
1152 got_packidx_close(p);
1153 } else
1154 *packidx = p;
1155 imsg_free(&imsg);
1156 return err;
1159 static const struct got_error *
1160 send_tree_enumeration_done(struct imsgbuf *ibuf)
1162 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1163 NULL, 0) == -1)
1164 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1166 return got_privsep_flush_imsg(ibuf);
1169 struct enumerated_tree {
1170 struct got_object_id id;
1171 char *path;
1172 uint8_t *buf;
1173 struct got_parsed_tree_entry *entries;
1174 int nentries;
1177 static const struct got_error *
1178 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1179 struct got_object_id *tree_id,
1180 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1181 struct got_object_cache *objcache, struct got_object_idset *idset,
1182 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1184 const struct got_error *err = NULL;
1185 struct got_object_id_queue ids;
1186 struct got_object_qid *qid;
1187 uint8_t *buf = NULL;
1188 struct got_parsed_tree_entry *entries = NULL;
1189 size_t nentries = 0, nentries_alloc = 0, i;
1190 struct enumerated_tree *tree;
1192 *ntrees = 0;
1193 *have_all_entries = 1;
1194 STAILQ_INIT(&ids);
1196 err = got_object_qid_alloc_partial(&qid);
1197 if (err)
1198 return err;
1199 memcpy(&qid->id, tree_id, sizeof(*tree_id));
1200 qid->data = strdup(path);
1201 if (qid->data == NULL) {
1202 err = got_error_from_errno("strdup");
1203 goto done;
1205 STAILQ_INSERT_TAIL(&ids, qid, entry);
1206 qid = NULL;
1208 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1209 do {
1210 const char *path;
1211 int idx, i;
1213 if (sigint_received) {
1214 err = got_error(GOT_ERR_CANCELLED);
1215 goto done;
1218 qid = STAILQ_FIRST(&ids);
1219 STAILQ_REMOVE_HEAD(&ids, entry);
1220 path = qid->data;
1222 idx = got_packidx_get_object_idx(packidx, &qid->id);
1223 if (idx == -1) {
1224 *have_all_entries = 0;
1225 break;
1228 err = open_tree(&buf, &entries, &nentries, &nentries_alloc,
1229 pack, packidx, idx, &qid->id, objcache);
1230 if (err) {
1231 if (err->code != GOT_ERR_NO_OBJ)
1232 goto done;
1235 err = got_object_idset_add(idset, &qid->id, NULL);
1236 if (err)
1237 goto done;
1239 for (i = 0; i < nentries; i++) {
1240 struct got_object_qid *eqid = NULL;
1241 struct got_parsed_tree_entry *pte = &entries[i];
1242 char *p;
1244 if (!S_ISDIR(pte->mode))
1245 continue;
1247 err = got_object_qid_alloc_partial(&eqid);
1248 if (err)
1249 goto done;
1250 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1252 if (got_object_idset_contains(idset, &eqid->id)) {
1253 got_object_qid_free(eqid);
1254 continue;
1257 if (asprintf(&p, "%s%s%s", path,
1258 got_path_is_root_dir(path) ? "" : "/",
1259 pte->name) == -1) {
1260 err = got_error_from_errno("asprintf");
1261 got_object_qid_free(eqid);
1262 goto done;
1264 eqid->data = p;
1265 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1268 if (*ntrees >= *nalloc) {
1269 struct enumerated_tree *new;
1270 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1271 sizeof(*new));
1272 if (new == NULL) {
1273 err = got_error_from_errno("malloc");
1274 goto done;
1276 *trees = new;
1277 *nalloc += 16;
1279 tree = &(*trees)[*ntrees];
1280 (*ntrees)++;
1281 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1282 tree->path = qid->data;
1283 tree->buf = buf;
1284 buf = NULL;
1285 tree->entries = entries;
1286 entries = NULL;
1287 nentries_alloc = 0;
1288 tree->nentries = nentries;
1289 nentries = 0;
1291 got_object_qid_free(qid);
1292 qid = NULL;
1293 } while (!STAILQ_EMPTY(&ids));
1295 if (*have_all_entries) {
1296 int i;
1298 * We have managed to traverse all entries in the hierarchy.
1299 * Tell the main process what we have found.
1301 for (i = 0; i < *ntrees; i++) {
1302 tree = &(*trees)[i];
1303 err = got_privsep_send_enumerated_tree(totlen,
1304 ibuf, &tree->id, tree->path, tree->entries,
1305 tree->nentries);
1306 if (err)
1307 goto done;
1308 free(tree->buf);
1309 tree->buf = NULL;
1310 free(tree->path);
1311 tree->path = NULL;
1312 free(tree->entries);
1313 tree->entries = NULL;
1315 *ntrees = 0; /* don't loop again below to free memory */
1317 err = send_tree_enumeration_done(ibuf);
1318 } else {
1320 * We can only load fully packed tree hierarchies on
1321 * behalf of the main process, otherwise the main process
1322 * gets a wrong idea about which tree objects have
1323 * already been traversed.
1324 * Indicate a missing entry for the root of this tree.
1325 * The main process should continue by loading this
1326 * entire tree the slow way.
1328 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1329 tree_id, "/", NULL, -1);
1330 if (err)
1331 goto done;
1333 done:
1334 free(buf);
1335 free(entries);
1336 for (i = 0; i < *ntrees; i++) {
1337 tree = &(*trees)[i];
1338 free(tree->buf);
1339 tree->buf = NULL;
1340 free(tree->path);
1341 tree->path = NULL;
1342 free(tree->entries);
1343 tree->entries = NULL;
1345 if (qid)
1346 free(qid->data);
1347 got_object_qid_free(qid);
1348 got_object_id_queue_free(&ids);
1349 if (err) {
1350 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1351 err = NULL;
1352 else
1353 got_privsep_send_error(ibuf, err);
1356 return err;
1359 static const struct got_error *
1360 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1361 struct got_pack *pack, struct got_packidx *packidx,
1362 struct got_object_cache *objcache)
1364 const struct got_error *err = NULL;
1365 struct got_object_id_queue commit_ids;
1366 const struct got_object_id_queue *parents = NULL;
1367 struct got_object_qid *qid = NULL;
1368 struct got_object *obj = NULL;
1369 struct got_commit_object *commit = NULL;
1370 struct got_object_id *tree_id = NULL;
1371 size_t totlen = 0;
1372 struct got_object_idset *idset, *queued_ids = NULL;
1373 int i, idx, have_all_entries = 1;
1374 struct enumerated_tree *trees = NULL;
1375 size_t ntrees = 0, nalloc = 16;
1377 STAILQ_INIT(&commit_ids);
1379 trees = calloc(nalloc, sizeof(*trees));
1380 if (trees == NULL)
1381 return got_error_from_errno("calloc");
1383 idset = got_object_idset_alloc();
1384 if (idset == NULL) {
1385 err = got_error_from_errno("got_object_idset_alloc");
1386 goto done;
1389 queued_ids = got_object_idset_alloc();
1390 if (queued_ids == NULL) {
1391 err = got_error_from_errno("got_object_idset_alloc");
1392 goto done;
1395 err = recv_object_id_queue(&commit_ids, queued_ids, ibuf);
1396 if (err)
1397 goto done;
1399 if (STAILQ_EMPTY(&commit_ids)) {
1400 err = got_error(GOT_ERR_PRIVSEP_MSG);
1401 goto done;
1404 err = recv_object_ids(idset, ibuf);
1405 if (err)
1406 goto done;
1408 while (!STAILQ_EMPTY(&commit_ids)) {
1409 if (sigint_received) {
1410 err = got_error(GOT_ERR_CANCELLED);
1411 goto done;
1414 qid = STAILQ_FIRST(&commit_ids);
1415 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1417 if (got_object_idset_contains(idset, &qid->id)) {
1418 got_object_qid_free(qid);
1419 qid = NULL;
1420 continue;
1423 idx = got_packidx_get_object_idx(packidx, &qid->id);
1424 if (idx == -1) {
1425 have_all_entries = 0;
1426 break;
1429 err = open_object(&obj, pack, packidx, idx, &qid->id,
1430 objcache);
1431 if (err)
1432 goto done;
1433 if (obj->type == GOT_OBJ_TYPE_TAG) {
1434 struct got_tag_object *tag;
1435 uint8_t *buf;
1436 size_t len;
1437 err = got_packfile_extract_object_to_mem(&buf,
1438 &len, obj, pack);
1439 if (err)
1440 goto done;
1441 obj->size = len;
1442 err = got_object_parse_tag(&tag, buf, len);
1443 if (err) {
1444 free(buf);
1445 goto done;
1447 idx = got_packidx_get_object_idx(packidx, &tag->id);
1448 if (idx == -1) {
1449 have_all_entries = 0;
1450 break;
1452 err = open_commit(&commit, pack, packidx, idx,
1453 &tag->id, objcache);
1454 got_object_tag_close(tag);
1455 free(buf);
1456 if (err)
1457 goto done;
1458 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1459 err = open_commit(&commit, pack, packidx, idx,
1460 &qid->id, objcache);
1461 if (err)
1462 goto done;
1463 } else {
1464 err = got_error(GOT_ERR_OBJ_TYPE);
1465 goto done;
1467 got_object_close(obj);
1468 obj = NULL;
1470 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1471 got_object_commit_get_committer_time(commit));
1472 if (err)
1473 goto done;
1475 tree_id = got_object_commit_get_tree_id(commit);
1476 idx = got_packidx_get_object_idx(packidx, tree_id);
1477 if (idx == -1) {
1478 have_all_entries = 0;
1479 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1480 tree_id, "/", NULL, -1);
1481 if (err)
1482 goto done;
1483 break;
1486 if (got_object_idset_contains(idset, tree_id)) {
1487 got_object_qid_free(qid);
1488 qid = NULL;
1489 err = send_tree_enumeration_done(ibuf);
1490 if (err)
1491 goto done;
1492 continue;
1495 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1496 tree_id, "/", pack, packidx, objcache, idset,
1497 &trees, &nalloc, &ntrees);
1498 if (err)
1499 goto done;
1501 if (!have_all_entries)
1502 break;
1504 got_object_qid_free(qid);
1505 qid = NULL;
1507 parents = got_object_commit_get_parent_ids(commit);
1508 if (parents) {
1509 struct got_object_qid *pid;
1510 STAILQ_FOREACH(pid, parents, entry) {
1511 if (got_object_idset_contains(idset, &pid->id))
1512 continue;
1513 if (got_object_idset_contains(queued_ids, &pid->id))
1514 continue;
1515 err = got_object_qid_alloc_partial(&qid);
1516 if (err)
1517 goto done;
1518 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1519 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1520 qid = NULL;
1524 got_object_commit_close(commit);
1525 commit = NULL;
1528 if (have_all_entries) {
1529 err = got_privsep_send_object_enumeration_done(ibuf);
1530 if (err)
1531 goto done;
1532 } else {
1533 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1534 if (err)
1535 goto done;
1537 done:
1538 if (obj)
1539 got_object_close(obj);
1540 if (commit)
1541 got_object_commit_close(commit);
1542 got_object_qid_free(qid);
1543 got_object_id_queue_free(&commit_ids);
1544 if (idset)
1545 got_object_idset_free(idset);
1546 if (queued_ids)
1547 got_object_idset_free(queued_ids);
1548 for (i = 0; i < ntrees; i++) {
1549 struct enumerated_tree *tree = &trees[i];
1550 free(tree->buf);
1551 free(tree->path);
1552 free(tree->entries);
1554 free(trees);
1555 return err;
1558 enum findtwixt_color {
1559 COLOR_KEEP = 0,
1560 COLOR_DROP,
1561 COLOR_SKIP,
1562 COLOR_MAX,
1565 static const struct got_error *
1566 paint_commit(struct got_object_qid *qid, intptr_t color)
1568 if (color < 0 || color >= COLOR_MAX)
1569 return got_error(GOT_ERR_RANGE);
1571 qid->data = (void *)color;
1572 return NULL;
1575 static const struct got_error *
1576 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1577 intptr_t color)
1579 const struct got_error *err;
1580 struct got_object_qid *qid;
1582 err = got_object_qid_alloc_partial(&qid);
1583 if (err)
1584 return err;
1586 memcpy(&qid->id, id, sizeof(qid->id));
1587 STAILQ_INSERT_TAIL(ids, qid, entry);
1588 return paint_commit(qid, color);
1591 static const struct got_error *
1592 paint_commits(struct got_object_id_queue *ids, int *nids,
1593 struct got_object_idset *keep, struct got_object_idset *drop,
1594 struct got_object_idset *skip, struct got_pack *pack,
1595 struct got_packidx *packidx, struct imsgbuf *ibuf,
1596 struct got_object_cache *objcache)
1598 const struct got_error *err = NULL;
1599 struct got_commit_object *commit = NULL;
1600 struct got_object_id_queue painted;
1601 const struct got_object_id_queue *parents;
1602 struct got_object_qid *qid = NULL;
1603 int nqueued = *nids, nskip = 0, npainted = 0;
1605 STAILQ_INIT(&painted);
1607 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1608 int idx;
1609 intptr_t color;
1611 if (sigint_received) {
1612 err = got_error(GOT_ERR_CANCELLED);
1613 goto done;
1616 qid = STAILQ_FIRST(ids);
1617 idx = got_packidx_get_object_idx(packidx, &qid->id);
1618 if (idx == -1) {
1619 qid = NULL;
1620 break;
1623 STAILQ_REMOVE_HEAD(ids, entry);
1624 nqueued--;
1625 color = (intptr_t)qid->data;
1626 if (color == COLOR_SKIP)
1627 nskip--;
1629 if (got_object_idset_contains(skip, &qid->id)) {
1630 got_object_qid_free(qid);
1631 qid = NULL;
1632 continue;
1635 switch (color) {
1636 case COLOR_KEEP:
1637 if (got_object_idset_contains(keep, &qid->id)) {
1638 got_object_qid_free(qid);
1639 qid = NULL;
1640 continue;
1642 if (got_object_idset_contains(drop, &qid->id)) {
1643 err = paint_commit(qid, COLOR_SKIP);
1644 if (err)
1645 goto done;
1647 err = got_object_idset_add(keep, &qid->id, NULL);
1648 if (err)
1649 goto done;
1650 break;
1651 case COLOR_DROP:
1652 if (got_object_idset_contains(drop, &qid->id)) {
1653 got_object_qid_free(qid);
1654 qid = NULL;
1655 continue;
1657 if (got_object_idset_contains(keep, &qid->id)) {
1658 err = paint_commit(qid, COLOR_SKIP);
1659 if (err)
1660 goto done;
1662 err = got_object_idset_add(drop, &qid->id, NULL);
1663 if (err)
1664 goto done;
1665 break;
1666 case COLOR_SKIP:
1667 if (!got_object_idset_contains(skip, &qid->id)) {
1668 err = got_object_idset_add(skip, &qid->id,
1669 NULL);
1670 if (err)
1671 goto done;
1673 break;
1674 default:
1675 /* should not happen */
1676 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1677 "%s invalid commit color %"PRIdPTR, __func__,
1678 color);
1679 goto done;
1682 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1683 objcache);
1684 if (err)
1685 goto done;
1687 parents = got_object_commit_get_parent_ids(commit);
1688 if (parents) {
1689 struct got_object_qid *pid;
1690 color = (intptr_t)qid->data;
1691 STAILQ_FOREACH(pid, parents, entry) {
1692 err = queue_commit_id(ids, &pid->id, color);
1693 if (err)
1694 goto done;
1695 nqueued++;
1696 if (color == COLOR_SKIP)
1697 nskip++;
1701 got_object_commit_close(commit);
1702 commit = NULL;
1704 STAILQ_INSERT_TAIL(&painted, qid, entry);
1705 qid = NULL;
1706 npainted++;
1708 err = got_privsep_send_painted_commits(ibuf, &painted,
1709 &npainted, 1, 0);
1710 if (err)
1711 goto done;
1714 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1715 if (err)
1716 goto done;
1718 *nids = nqueued;
1719 done:
1720 if (commit)
1721 got_object_commit_close(commit);
1722 got_object_qid_free(qid);
1723 return err;
1726 static void
1727 commit_painting_free(struct got_object_idset **keep,
1728 struct got_object_idset **drop,
1729 struct got_object_idset **skip)
1731 if (*keep) {
1732 got_object_idset_free(*keep);
1733 *keep = NULL;
1735 if (*drop) {
1736 got_object_idset_free(*drop);
1737 *drop = NULL;
1739 if (*skip) {
1740 got_object_idset_free(*skip);
1741 *skip = NULL;
1745 static const struct got_error *
1746 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1747 struct got_object_idset **drop, struct got_object_idset **skip)
1749 const struct got_error *err = NULL;
1751 *keep = got_object_idset_alloc();
1752 if (*keep == NULL) {
1753 err = got_error_from_errno("got_object_idset_alloc");
1754 goto done;
1756 *drop = got_object_idset_alloc();
1757 if (*drop == NULL) {
1758 err = got_error_from_errno("got_object_idset_alloc");
1759 goto done;
1761 *skip = got_object_idset_alloc();
1762 if (*skip == NULL) {
1763 err = got_error_from_errno("got_object_idset_alloc");
1764 goto done;
1767 err = recv_object_ids(*keep, ibuf);
1768 if (err)
1769 goto done;
1770 err = recv_object_ids(*drop, ibuf);
1771 if (err)
1772 goto done;
1773 err = recv_object_ids(*skip, ibuf);
1774 if (err)
1775 goto done;
1777 done:
1778 if (err)
1779 commit_painting_free(keep, drop, skip);
1781 return err;
1784 static const struct got_error *
1785 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1786 struct got_pack *pack, struct got_packidx *packidx,
1787 struct got_object_cache *objcache, struct got_object_idset *keep,
1788 struct got_object_idset *drop, struct got_object_idset *skip)
1790 const struct got_error *err = NULL;
1791 struct got_imsg_commit_painting_request ireq;
1792 struct got_object_id id;
1793 size_t datalen;
1794 struct got_object_id_queue ids;
1795 int nids = 0;
1797 STAILQ_INIT(&ids);
1799 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1800 if (datalen != sizeof(ireq))
1801 return got_error(GOT_ERR_PRIVSEP_LEN);
1802 memcpy(&ireq, imsg->data, sizeof(ireq));
1803 memcpy(&id, &ireq.id, sizeof(id));
1805 err = queue_commit_id(&ids, &id, ireq.color);
1806 if (err)
1807 return err;
1808 nids = 1;
1810 err = paint_commits(&ids, &nids, keep, drop, skip,
1811 pack, packidx, ibuf, objcache);
1812 if (err)
1813 goto done;
1815 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1816 if (err)
1817 goto done;
1819 err = got_privsep_send_painting_commits_done(ibuf);
1820 done:
1821 got_object_id_queue_free(&ids);
1822 return err;
1825 static const struct got_error *
1826 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1828 const struct got_error *err = NULL;
1829 struct imsg imsg;
1830 struct got_imsg_pack ipack;
1831 size_t datalen;
1832 struct got_pack *pack;
1834 *packp = NULL;
1836 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1837 if (err)
1838 return err;
1840 pack = calloc(1, sizeof(*pack));
1841 if (pack == NULL) {
1842 err = got_error_from_errno("calloc");
1843 goto done;
1846 if (imsg.hdr.type != GOT_IMSG_PACK) {
1847 err = got_error(GOT_ERR_PRIVSEP_MSG);
1848 goto done;
1851 if (imsg.fd == -1) {
1852 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1853 goto done;
1856 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1857 if (datalen != sizeof(ipack)) {
1858 err = got_error(GOT_ERR_PRIVSEP_LEN);
1859 goto done;
1861 memcpy(&ipack, imsg.data, sizeof(ipack));
1863 pack->filesize = ipack.filesize;
1864 pack->fd = dup(imsg.fd);
1865 if (pack->fd == -1) {
1866 err = got_error_from_errno("dup");
1867 goto done;
1869 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1870 err = got_error_from_errno("lseek");
1871 goto done;
1873 pack->path_packfile = strdup(ipack.path_packfile);
1874 if (pack->path_packfile == NULL) {
1875 err = got_error_from_errno("strdup");
1876 goto done;
1879 err = got_delta_cache_alloc(&pack->delta_cache);
1880 if (err)
1881 goto done;
1883 #ifndef GOT_PACK_NO_MMAP
1884 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1885 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1886 pack->fd, 0);
1887 if (pack->map == MAP_FAILED)
1888 pack->map = NULL; /* fall back to read(2) */
1890 #endif
1891 done:
1892 if (err) {
1893 if (imsg.fd != -1)
1894 close(imsg.fd);
1895 free(pack);
1896 } else
1897 *packp = pack;
1898 imsg_free(&imsg);
1899 return err;
1902 int
1903 main(int argc, char *argv[])
1905 const struct got_error *err = NULL;
1906 struct imsgbuf ibuf;
1907 struct imsg imsg;
1908 struct got_packidx *packidx = NULL;
1909 struct got_pack *pack = NULL;
1910 struct got_object_cache objcache;
1911 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1912 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1913 struct got_parsed_tree_entry *entries = NULL;
1914 size_t nentries = 0, nentries_alloc = 0;
1916 //static int attached;
1917 //while (!attached) sleep(1);
1919 signal(SIGINT, catch_sigint);
1921 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1923 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1924 if (err) {
1925 err = got_error_from_errno("got_object_cache_init");
1926 got_privsep_send_error(&ibuf, err);
1927 return 1;
1930 #ifndef PROFILE
1931 /* revoke access to most system calls */
1932 if (pledge("stdio recvfd", NULL) == -1) {
1933 err = got_error_from_errno("pledge");
1934 got_privsep_send_error(&ibuf, err);
1935 return 1;
1937 #endif
1939 err = receive_packidx(&packidx, &ibuf);
1940 if (err) {
1941 got_privsep_send_error(&ibuf, err);
1942 return 1;
1945 err = receive_pack(&pack, &ibuf);
1946 if (err) {
1947 got_privsep_send_error(&ibuf, err);
1948 return 1;
1951 for (;;) {
1952 imsg.fd = -1;
1954 if (sigint_received) {
1955 err = got_error(GOT_ERR_CANCELLED);
1956 break;
1959 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1960 if (err) {
1961 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1962 err = NULL;
1963 break;
1966 if (imsg.hdr.type == GOT_IMSG_STOP)
1967 break;
1969 switch (imsg.hdr.type) {
1970 case GOT_IMSG_TMPFD:
1971 if (basefile == NULL) {
1972 err = receive_tempfile(&basefile, "w+",
1973 &imsg, &ibuf);
1974 } else if (accumfile == NULL) {
1975 err = receive_tempfile(&accumfile, "w+",
1976 &imsg, &ibuf);
1977 } else
1978 err = got_error(GOT_ERR_PRIVSEP_MSG);
1979 break;
1980 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1981 err = object_request(&imsg, &ibuf, pack, packidx,
1982 &objcache);
1983 break;
1984 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1985 if (basefile == NULL || accumfile == NULL) {
1986 err = got_error(GOT_ERR_PRIVSEP_MSG);
1987 break;
1989 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1990 &objcache, basefile, accumfile);
1991 break;
1992 case GOT_IMSG_RAW_DELTA_OUTFD:
1993 if (delta_outfile != NULL) {
1994 err = got_error(GOT_ERR_PRIVSEP_MSG);
1995 break;
1997 err = receive_tempfile(&delta_outfile, "w",
1998 &imsg, &ibuf);
1999 break;
2000 case GOT_IMSG_RAW_DELTA_REQUEST:
2001 if (delta_outfile == NULL) {
2002 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2003 break;
2005 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2006 pack, packidx);
2007 break;
2008 case GOT_IMSG_DELTA_REUSE_REQUEST:
2009 err = delta_reuse_request(&imsg, &ibuf, pack, packidx);
2010 break;
2011 case GOT_IMSG_COMMIT_REQUEST:
2012 err = commit_request(&imsg, &ibuf, pack, packidx,
2013 &objcache);
2014 break;
2015 case GOT_IMSG_TREE_REQUEST:
2016 err = tree_request(&imsg, &ibuf, pack, packidx,
2017 &objcache, &entries, &nentries, &nentries_alloc);
2018 break;
2019 case GOT_IMSG_BLOB_REQUEST:
2020 if (basefile == NULL || accumfile == NULL) {
2021 err = got_error(GOT_ERR_PRIVSEP_MSG);
2022 break;
2024 err = blob_request(&imsg, &ibuf, pack, packidx,
2025 &objcache, basefile, accumfile);
2026 break;
2027 case GOT_IMSG_TAG_REQUEST:
2028 err = tag_request(&imsg, &ibuf, pack, packidx,
2029 &objcache);
2030 break;
2031 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2032 err = commit_traversal_request(&imsg, &ibuf, pack,
2033 packidx, &objcache);
2034 break;
2035 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2036 err = enumeration_request(&imsg, &ibuf, pack,
2037 packidx, &objcache);
2038 break;
2039 case GOT_IMSG_COMMIT_PAINTING_INIT:
2040 commit_painting_free(&keep, &drop, &skip);
2041 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2042 break;
2043 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2044 if (keep == NULL || drop == NULL || skip == NULL) {
2045 err = got_error(GOT_ERR_PRIVSEP_MSG);
2046 break;
2048 err = commit_painting_request(&imsg, &ibuf, pack,
2049 packidx, &objcache, keep, drop, skip);
2050 break;
2051 case GOT_IMSG_COMMIT_PAINTING_DONE:
2052 commit_painting_free(&keep, &drop, &skip);
2053 break;
2054 default:
2055 err = got_error(GOT_ERR_PRIVSEP_MSG);
2056 break;
2059 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
2060 err = got_error_from_errno("close");
2061 imsg_free(&imsg);
2062 if (err)
2063 break;
2066 free(entries);
2067 commit_painting_free(&keep, &drop, &skip);
2068 if (packidx)
2069 got_packidx_close(packidx);
2070 if (pack)
2071 got_pack_close(pack);
2072 got_object_cache_close(&objcache);
2073 imsg_clear(&ibuf);
2074 if (basefile && fclose(basefile) == EOF && err == NULL)
2075 err = got_error_from_errno("fclose");
2076 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2077 err = got_error_from_errno("fclose");
2078 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2079 err = got_error_from_errno("fclose");
2080 if (err) {
2081 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2082 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2083 got_privsep_send_error(&ibuf, err);
2086 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2087 err = got_error_from_errno("close");
2088 return err ? 1 : 0;