2 5aa81393 2020-01-06 stsp * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
4 876c234b 2018-09-10 stsp * Permission to use, copy, modify, and distribute this software for any
5 876c234b 2018-09-10 stsp * purpose with or without fee is hereby granted, provided that the above
6 876c234b 2018-09-10 stsp * copyright notice and this permission notice appear in all copies.
8 876c234b 2018-09-10 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 876c234b 2018-09-10 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 876c234b 2018-09-10 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 876c234b 2018-09-10 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 876c234b 2018-09-10 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 876c234b 2018-09-10 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 876c234b 2018-09-10 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 0ab4c957 2022-06-13 stsp #include <sys/stat.h>
18 876c234b 2018-09-10 stsp #include <sys/types.h>
19 876c234b 2018-09-10 stsp #include <sys/queue.h>
20 876c234b 2018-09-10 stsp #include <sys/uio.h>
21 876c234b 2018-09-10 stsp #include <sys/time.h>
22 876c234b 2018-09-10 stsp #include <sys/mman.h>
24 756050ac 2022-07-19 op #include <inttypes.h>
25 876c234b 2018-09-10 stsp #include <limits.h>
26 99437157 2018-11-11 stsp #include <signal.h>
27 876c234b 2018-09-10 stsp #include <stdint.h>
28 876c234b 2018-09-10 stsp #include <imsg.h>
29 876c234b 2018-09-10 stsp #include <stdio.h>
30 876c234b 2018-09-10 stsp #include <stdlib.h>
31 876c234b 2018-09-10 stsp #include <string.h>
32 876c234b 2018-09-10 stsp #include <sha1.h>
33 5822e79e 2023-02-23 op #include <sha2.h>
34 81a12da5 2020-09-09 naddy #include <unistd.h>
35 876c234b 2018-09-10 stsp #include <zlib.h>
37 876c234b 2018-09-10 stsp #include "got_error.h"
38 876c234b 2018-09-10 stsp #include "got_object.h"
39 3022d272 2019-11-14 stsp #include "got_path.h"
41 876c234b 2018-09-10 stsp #include "got_lib_delta.h"
42 ab2f42e7 2019-11-10 stsp #include "got_lib_delta_cache.h"
43 876c234b 2018-09-10 stsp #include "got_lib_object.h"
44 2f43cd69 2023-04-14 stsp #include "got_lib_object_qid.h"
45 c59b3346 2018-09-11 stsp #include "got_lib_object_cache.h"
46 876c234b 2018-09-10 stsp #include "got_lib_object_parse.h"
47 fae7e038 2022-05-07 stsp #include "got_lib_object_idset.h"
48 876c234b 2018-09-10 stsp #include "got_lib_privsep.h"
49 876c234b 2018-09-10 stsp #include "got_lib_pack.h"
51 61af9b21 2022-06-28 stsp #ifndef nitems
52 61af9b21 2022-06-28 stsp #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
55 99437157 2018-11-11 stsp static volatile sig_atomic_t sigint_received;
58 99437157 2018-11-11 stsp catch_sigint(int signo)
60 99437157 2018-11-11 stsp sigint_received = 1;
63 876c234b 2018-09-10 stsp static const struct got_error *
64 704b89c4 2019-05-23 stsp open_object(struct got_object **obj, struct got_pack *pack,
65 704b89c4 2019-05-23 stsp struct got_packidx *packidx, int idx, struct got_object_id *id,
66 704b89c4 2019-05-23 stsp struct got_object_cache *objcache)
68 704b89c4 2019-05-23 stsp const struct got_error *err;
70 704b89c4 2019-05-23 stsp err = got_packfile_open_object(obj, pack, packidx, idx, id);
73 704b89c4 2019-05-23 stsp (*obj)->refcnt++;
75 704b89c4 2019-05-23 stsp err = got_object_cache_add(objcache, id, *obj);
77 79c99a64 2019-05-23 stsp if (err->code == GOT_ERR_OBJ_EXISTS ||
78 79c99a64 2019-05-23 stsp err->code == GOT_ERR_OBJ_TOO_LARGE)
82 704b89c4 2019-05-23 stsp (*obj)->refcnt++;
83 704b89c4 2019-05-23 stsp return NULL;
86 704b89c4 2019-05-23 stsp static const struct got_error *
87 876c234b 2018-09-10 stsp object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
88 c59b3346 2018-09-11 stsp struct got_packidx *packidx, struct got_object_cache *objcache)
90 876c234b 2018-09-10 stsp const struct got_error *err = NULL;
91 876c234b 2018-09-10 stsp struct got_imsg_packed_object iobj;
92 876c234b 2018-09-10 stsp struct got_object *obj;
93 106807b4 2018-09-15 stsp struct got_object_id id;
94 876c234b 2018-09-10 stsp size_t datalen;
96 876c234b 2018-09-10 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
97 876c234b 2018-09-10 stsp if (datalen != sizeof(iobj))
98 876c234b 2018-09-10 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
99 876c234b 2018-09-10 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
100 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
102 704b89c4 2019-05-23 stsp obj = got_object_cache_get(objcache, &id);
104 704b89c4 2019-05-23 stsp obj->refcnt++;
106 704b89c4 2019-05-23 stsp err = open_object(&obj, pack, packidx, iobj.idx, &id,
112 876c234b 2018-09-10 stsp err = got_privsep_send_obj(ibuf, obj);
114 876c234b 2018-09-10 stsp got_object_close(obj);
115 876c234b 2018-09-10 stsp return err;
118 50127d69 2021-09-25 stsp static const struct got_error *
119 ca6e02ac 2020-01-07 stsp open_commit(struct got_commit_object **commit, struct got_pack *pack,
120 ca6e02ac 2020-01-07 stsp struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
121 ca6e02ac 2020-01-07 stsp struct got_object_cache *objcache)
123 cfd633c2 2018-09-10 stsp const struct got_error *err = NULL;
124 cb5e38fd 2019-05-23 stsp struct got_object *obj = NULL;
125 cb5e38fd 2019-05-23 stsp uint8_t *buf = NULL;
126 cfd633c2 2018-09-10 stsp size_t len;
128 ca6e02ac 2020-01-07 stsp *commit = NULL;
130 ca6e02ac 2020-01-07 stsp obj = got_object_cache_get(objcache, id);
132 704b89c4 2019-05-23 stsp obj->refcnt++;
134 ca6e02ac 2020-01-07 stsp err = open_object(&obj, pack, packidx, obj_idx, id,
137 704b89c4 2019-05-23 stsp return err;
140 cfd633c2 2018-09-10 stsp err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
144 cfd633c2 2018-09-10 stsp obj->size = len;
146 ca6e02ac 2020-01-07 stsp err = got_object_parse_commit(commit, buf, len);
148 ca6e02ac 2020-01-07 stsp got_object_close(obj);
150 ca6e02ac 2020-01-07 stsp return err;
153 ca6e02ac 2020-01-07 stsp static const struct got_error *
154 ca6e02ac 2020-01-07 stsp commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
155 ca6e02ac 2020-01-07 stsp struct got_packidx *packidx, struct got_object_cache *objcache)
157 ca6e02ac 2020-01-07 stsp const struct got_error *err = NULL;
158 ca6e02ac 2020-01-07 stsp struct got_imsg_packed_object iobj;
159 ca6e02ac 2020-01-07 stsp struct got_commit_object *commit = NULL;
160 ca6e02ac 2020-01-07 stsp struct got_object_id id;
161 ca6e02ac 2020-01-07 stsp size_t datalen;
163 ca6e02ac 2020-01-07 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
164 ca6e02ac 2020-01-07 stsp if (datalen != sizeof(iobj))
165 ca6e02ac 2020-01-07 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
166 ca6e02ac 2020-01-07 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
167 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
169 ca6e02ac 2020-01-07 stsp err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
173 cfd633c2 2018-09-10 stsp err = got_privsep_send_commit(ibuf, commit);
175 cb5e38fd 2019-05-23 stsp if (commit)
176 cb5e38fd 2019-05-23 stsp got_object_commit_close(commit);
178 7762fe12 2018-11-05 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
179 7762fe12 2018-11-05 stsp err = NULL;
181 7762fe12 2018-11-05 stsp got_privsep_send_error(ibuf, err);
184 7762fe12 2018-11-05 stsp return err;
187 50127d69 2021-09-25 stsp static const struct got_error *
188 cdfe5501 2023-04-24 stsp open_tree(uint8_t **buf, size_t *len,
189 cdfe5501 2023-04-24 stsp struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
190 cdfe5501 2023-04-24 stsp struct got_object_id *id, struct got_object_cache *objcache)
192 ca6e02ac 2020-01-07 stsp const struct got_error *err = NULL;
193 ca6e02ac 2020-01-07 stsp struct got_object *obj = NULL;
194 cdfe5501 2023-04-24 stsp int cached = 0;
196 ca6e02ac 2020-01-07 stsp *buf = NULL;
199 ca6e02ac 2020-01-07 stsp obj = got_object_cache_get(objcache, id);
201 ca6e02ac 2020-01-07 stsp obj->refcnt++;
202 cdfe5501 2023-04-24 stsp cached = 1;
204 ca6e02ac 2020-01-07 stsp err = open_object(&obj, pack, packidx, obj_idx, id,
207 ca6e02ac 2020-01-07 stsp return err;
210 cdfe5501 2023-04-24 stsp err = got_packfile_extract_object_to_mem(buf, len, obj, pack);
214 cdfe5501 2023-04-24 stsp if (!cached)
215 cdfe5501 2023-04-24 stsp obj->size = *len;
217 ca6e02ac 2020-01-07 stsp got_object_close(obj);
219 ca6e02ac 2020-01-07 stsp free(*buf);
220 ca6e02ac 2020-01-07 stsp *buf = NULL;
222 ca6e02ac 2020-01-07 stsp return err;
225 7762fe12 2018-11-05 stsp static const struct got_error *
226 876c234b 2018-09-10 stsp tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
227 d294b1dc 2022-10-18 stsp struct got_packidx *packidx, struct got_object_cache *objcache,
228 d294b1dc 2022-10-18 stsp struct got_parsed_tree_entry **entries, size_t *nentries,
229 d294b1dc 2022-10-18 stsp size_t *nentries_alloc)
231 e7885405 2018-09-10 stsp const struct got_error *err = NULL;
232 13c729f7 2018-12-24 stsp struct got_imsg_packed_object iobj;
233 cb5e38fd 2019-05-23 stsp uint8_t *buf = NULL;
234 cdfe5501 2023-04-24 stsp size_t len = 0;
235 13c729f7 2018-12-24 stsp struct got_object_id id;
236 13c729f7 2018-12-24 stsp size_t datalen;
238 13c729f7 2018-12-24 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
239 13c729f7 2018-12-24 stsp if (datalen != sizeof(iobj))
240 13c729f7 2018-12-24 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
241 13c729f7 2018-12-24 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
242 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
244 cdfe5501 2023-04-24 stsp err = open_tree(&buf, &len, pack, packidx, iobj.idx, &id, objcache);
246 ca6e02ac 2020-01-07 stsp return err;
248 cdfe5501 2023-04-24 stsp err = got_object_parse_tree(entries, nentries, nentries_alloc,
253 d294b1dc 2022-10-18 stsp err = got_privsep_send_tree(ibuf, *entries, *nentries);
255 e7885405 2018-09-10 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
256 e7885405 2018-09-10 stsp err = NULL;
258 e7885405 2018-09-10 stsp got_privsep_send_error(ibuf, err);
262 e7885405 2018-09-10 stsp return err;
265 876c234b 2018-09-10 stsp static const struct got_error *
266 030daac8 2021-09-25 stsp receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
268 3840f4c9 2018-09-12 stsp const struct got_error *err;
269 3840f4c9 2018-09-12 stsp struct imsg imsg;
270 55da3778 2018-09-10 stsp size_t datalen;
273 3840f4c9 2018-09-12 stsp err = got_privsep_recv_imsg(&imsg, ibuf, 0);
275 55da3778 2018-09-10 stsp return err;
277 3840f4c9 2018-09-12 stsp if (imsg.hdr.type != imsg_code) {
278 55da3778 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
282 3840f4c9 2018-09-12 stsp datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
283 55da3778 2018-09-10 stsp if (datalen != 0) {
284 55da3778 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
287 2c52c623 2024-01-30 op fd = imsg_get_fd(&imsg);
288 2c52c623 2024-01-30 op if (fd == -1) {
289 55da3778 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_NO_FD);
293 2c52c623 2024-01-30 op *f = fdopen(fd, "w+");
294 3840f4c9 2018-09-12 stsp if (*f == NULL) {
295 638f9024 2019-05-13 stsp err = got_error_from_errno("fdopen");
300 3840f4c9 2018-09-12 stsp imsg_free(&imsg);
301 3840f4c9 2018-09-12 stsp return err;
304 db696021 2022-01-04 stsp static const struct got_error *
305 67fd6849 2022-02-13 stsp receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
306 db696021 2022-01-04 stsp struct imsgbuf *ibuf)
308 2c52c623 2024-01-30 op const struct got_error *err;
309 db696021 2022-01-04 stsp size_t datalen;
312 db696021 2022-01-04 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
313 db696021 2022-01-04 stsp if (datalen != 0)
314 db696021 2022-01-04 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
316 2c52c623 2024-01-30 op fd = imsg_get_fd(imsg);
317 2c52c623 2024-01-30 op if (fd == -1)
318 db696021 2022-01-04 stsp return got_error(GOT_ERR_PRIVSEP_NO_FD);
320 2c52c623 2024-01-30 op *f = fdopen(fd, mode);
321 2c52c623 2024-01-30 op if (*f == NULL) {
322 2c52c623 2024-01-30 op err = got_error_from_errno("fdopen");
327 db696021 2022-01-04 stsp return NULL;
330 3840f4c9 2018-09-12 stsp static const struct got_error *
331 3840f4c9 2018-09-12 stsp blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
332 db696021 2022-01-04 stsp struct got_packidx *packidx, struct got_object_cache *objcache,
333 db696021 2022-01-04 stsp FILE *basefile, FILE *accumfile)
335 3840f4c9 2018-09-12 stsp const struct got_error *err = NULL;
336 ebc55e2d 2018-12-24 stsp struct got_imsg_packed_object iobj;
337 3840f4c9 2018-09-12 stsp struct got_object *obj = NULL;
338 db696021 2022-01-04 stsp FILE *outfile = NULL;
339 ebc55e2d 2018-12-24 stsp struct got_object_id id;
340 ebc55e2d 2018-12-24 stsp size_t datalen;
341 ac544f8c 2019-01-13 stsp uint64_t blob_size;
342 ac544f8c 2019-01-13 stsp uint8_t *buf = NULL;
344 ebc55e2d 2018-12-24 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
345 ebc55e2d 2018-12-24 stsp if (datalen != sizeof(iobj))
346 ebc55e2d 2018-12-24 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
347 ebc55e2d 2018-12-24 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
348 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
350 704b89c4 2019-05-23 stsp obj = got_object_cache_get(objcache, &id);
352 704b89c4 2019-05-23 stsp obj->refcnt++;
354 704b89c4 2019-05-23 stsp err = open_object(&obj, pack, packidx, iobj.idx, &id,
357 704b89c4 2019-05-23 stsp return err;
360 3840f4c9 2018-09-12 stsp err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
364 ac544f8c 2019-01-13 stsp if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
365 42c69117 2019-11-10 stsp err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
369 ac544f8c 2019-01-13 stsp blob_size = obj->size;
371 ac544f8c 2019-01-13 stsp if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
372 ac544f8c 2019-01-13 stsp err = got_packfile_extract_object_to_mem(&buf, &obj->size,
373 ac544f8c 2019-01-13 stsp obj, pack);
375 ac544f8c 2019-01-13 stsp err = got_packfile_extract_object(pack, obj, outfile, basefile,
376 ac544f8c 2019-01-13 stsp accumfile);
380 ac544f8c 2019-01-13 stsp err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
383 56b63ca4 2021-01-22 stsp if (outfile && fclose(outfile) == EOF && err == NULL)
384 638f9024 2019-05-13 stsp err = got_error_from_errno("fclose");
385 cb5e38fd 2019-05-23 stsp got_object_close(obj);
386 3840f4c9 2018-09-12 stsp if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
387 3840f4c9 2018-09-12 stsp got_privsep_send_error(ibuf, err);
389 55da3778 2018-09-10 stsp return err;
392 876c234b 2018-09-10 stsp static const struct got_error *
393 f4a881ce 2018-11-17 stsp tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
394 f4a881ce 2018-11-17 stsp struct got_packidx *packidx, struct got_object_cache *objcache)
396 f4a881ce 2018-11-17 stsp const struct got_error *err = NULL;
397 268f7291 2018-12-24 stsp struct got_imsg_packed_object iobj;
398 f4a881ce 2018-11-17 stsp struct got_object *obj = NULL;
399 f4a881ce 2018-11-17 stsp struct got_tag_object *tag = NULL;
400 cb5e38fd 2019-05-23 stsp uint8_t *buf = NULL;
401 f4a881ce 2018-11-17 stsp size_t len;
402 268f7291 2018-12-24 stsp struct got_object_id id;
403 268f7291 2018-12-24 stsp size_t datalen;
405 268f7291 2018-12-24 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
406 268f7291 2018-12-24 stsp if (datalen != sizeof(iobj))
407 268f7291 2018-12-24 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
408 268f7291 2018-12-24 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
409 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
411 704b89c4 2019-05-23 stsp obj = got_object_cache_get(objcache, &id);
413 704b89c4 2019-05-23 stsp obj->refcnt++;
415 704b89c4 2019-05-23 stsp err = open_object(&obj, pack, packidx, iobj.idx, &id,
418 704b89c4 2019-05-23 stsp return err;
421 f4a881ce 2018-11-17 stsp err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
425 f4a881ce 2018-11-17 stsp obj->size = len;
426 f4a881ce 2018-11-17 stsp err = got_object_parse_tag(&tag, buf, len);
430 f4a881ce 2018-11-17 stsp err = got_privsep_send_tag(ibuf, tag);
433 cb5e38fd 2019-05-23 stsp got_object_close(obj);
435 cb5e38fd 2019-05-23 stsp got_object_tag_close(tag);
437 ca6e02ac 2020-01-07 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
438 ca6e02ac 2020-01-07 stsp err = NULL;
440 ca6e02ac 2020-01-07 stsp got_privsep_send_error(ibuf, err);
443 ca6e02ac 2020-01-07 stsp return err;
446 50127d69 2021-09-25 stsp static const struct got_error *
447 cdfe5501 2023-04-24 stsp tree_path_changed(int *changed, uint8_t **buf1, size_t *len1,
448 cdfe5501 2023-04-24 stsp uint8_t **buf2, size_t *len2, const char *path,
449 cdfe5501 2023-04-24 stsp struct got_pack *pack, struct got_packidx *packidx,
450 ca6e02ac 2020-01-07 stsp struct imsgbuf *ibuf, struct got_object_cache *objcache)
452 ca6e02ac 2020-01-07 stsp const struct got_error *err = NULL;
453 cdfe5501 2023-04-24 stsp struct got_parsed_tree_entry pte1, pte2;
454 ca6e02ac 2020-01-07 stsp const char *seg, *s;
455 ca6e02ac 2020-01-07 stsp size_t seglen;
456 cdfe5501 2023-04-24 stsp size_t remain1 = *len1, remain2 = *len2, elen;
457 cdfe5501 2023-04-24 stsp uint8_t *next_entry1 = *buf1;
458 cdfe5501 2023-04-24 stsp uint8_t *next_entry2 = *buf2;
460 cdfe5501 2023-04-24 stsp memset(&pte1, 0, sizeof(pte1));
461 cdfe5501 2023-04-24 stsp memset(&pte2, 0, sizeof(pte2));
463 ca6e02ac 2020-01-07 stsp *changed = 0;
465 ca6e02ac 2020-01-07 stsp /* We not do support comparing the root path. */
466 61a7d79f 2020-02-29 stsp if (got_path_is_root_dir(path))
467 63f810e6 2020-02-29 stsp return got_error_path(path, GOT_ERR_BAD_PATH);
470 61a7d79f 2020-02-29 stsp while (*s == '/')
473 ca6e02ac 2020-01-07 stsp seglen = 0;
474 ca6e02ac 2020-01-07 stsp while (*s) {
475 ca6e02ac 2020-01-07 stsp if (*s != '/') {
483 cdfe5501 2023-04-24 stsp * As an optimization we compare entries in on-disk order
484 cdfe5501 2023-04-24 stsp * rather than in got_path_cmp() order. We only need to
485 cdfe5501 2023-04-24 stsp * find out if any entries differ. Parsing all entries and
486 cdfe5501 2023-04-24 stsp * sorting them slows us down significantly when tree objects
487 cdfe5501 2023-04-24 stsp * have thousands of entries. We can assume that on-disk entry
488 cdfe5501 2023-04-24 stsp * ordering is stable, as per got_object_tree_create() and
489 cdfe5501 2023-04-24 stsp * sort_tree_entries_the_way_git_likes_it(). Other orderings
490 cdfe5501 2023-04-24 stsp * are incompatible with Git and would yield false positives
491 cdfe5501 2023-04-24 stsp * here, too.
493 cdfe5501 2023-04-24 stsp while (remain1 > 0) {
494 cdfe5501 2023-04-24 stsp err = got_object_parse_tree_entry(&pte1, &elen,
495 cdfe5501 2023-04-24 stsp next_entry1, remain1);
497 cdfe5501 2023-04-24 stsp return err;
498 cdfe5501 2023-04-24 stsp next_entry1 += elen;
499 cdfe5501 2023-04-24 stsp remain1 -= elen;
500 cdfe5501 2023-04-24 stsp if (strncmp(pte1.name, seg, seglen) != 0 ||
501 cdfe5501 2023-04-24 stsp pte1.name[seglen] != '\0') {
502 cdfe5501 2023-04-24 stsp memset(&pte1, 0, sizeof(pte1));
507 cdfe5501 2023-04-24 stsp if (pte1.name == NULL) {
508 ca6e02ac 2020-01-07 stsp err = got_error(GOT_ERR_NO_OBJ);
512 cdfe5501 2023-04-24 stsp if (remain2 == 0) {
513 ca6e02ac 2020-01-07 stsp *changed = 1;
517 cdfe5501 2023-04-24 stsp while (remain2 > 0) {
518 cdfe5501 2023-04-24 stsp err = got_object_parse_tree_entry(&pte2, &elen,
519 cdfe5501 2023-04-24 stsp next_entry2, remain2);
521 cdfe5501 2023-04-24 stsp return err;
522 cdfe5501 2023-04-24 stsp next_entry2 += elen;
523 cdfe5501 2023-04-24 stsp remain2 -= elen;
524 cdfe5501 2023-04-24 stsp if (strncmp(pte2.name, seg, seglen) != 0 ||
525 cdfe5501 2023-04-24 stsp pte2.name[seglen] != '\0') {
526 cdfe5501 2023-04-24 stsp memset(&pte2, 0, sizeof(pte2));
532 cdfe5501 2023-04-24 stsp if (pte2.name == NULL) {
533 ca6e02ac 2020-01-07 stsp *changed = 1;
537 cdfe5501 2023-04-24 stsp if (pte1.mode != pte2.mode) {
538 cdfe5501 2023-04-24 stsp *changed = 1;
542 cdfe5501 2023-04-24 stsp if (memcmp(pte1.id, pte2.id, SHA1_DIGEST_LENGTH) == 0) {
543 ca6e02ac 2020-01-07 stsp *changed = 0;
547 ca6e02ac 2020-01-07 stsp if (*s == '\0') { /* final path element */
548 ca6e02ac 2020-01-07 stsp *changed = 1;
552 ca6e02ac 2020-01-07 stsp seg = s + 1;
554 ca6e02ac 2020-01-07 stsp seglen = 0;
556 ca6e02ac 2020-01-07 stsp struct got_object_id id1, id2;
559 cdfe5501 2023-04-24 stsp memcpy(id1.sha1, pte1.id, SHA1_DIGEST_LENGTH);
560 00927983 2020-04-19 stsp idx = got_packidx_get_object_idx(packidx, &id1);
561 ca6e02ac 2020-01-07 stsp if (idx == -1) {
562 ded8fbb8 2020-04-19 stsp err = got_error_no_obj(&id1);
565 ca6e02ac 2020-01-07 stsp free(*buf1);
566 ca6e02ac 2020-01-07 stsp *buf1 = NULL;
567 cdfe5501 2023-04-24 stsp err = open_tree(buf1, len1, pack, packidx, idx, &id1,
569 cdfe5501 2023-04-24 stsp memset(&pte1, 0, sizeof(pte1));
572 cdfe5501 2023-04-24 stsp next_entry1 = *buf1;
573 cdfe5501 2023-04-24 stsp remain1 = *len1;
575 cdfe5501 2023-04-24 stsp memcpy(id2.sha1, pte2.id, SHA1_DIGEST_LENGTH);
576 00927983 2020-04-19 stsp idx = got_packidx_get_object_idx(packidx, &id2);
577 ca6e02ac 2020-01-07 stsp if (idx == -1) {
578 ded8fbb8 2020-04-19 stsp err = got_error_no_obj(&id2);
581 ca6e02ac 2020-01-07 stsp free(*buf2);
582 ca6e02ac 2020-01-07 stsp *buf2 = NULL;
583 cdfe5501 2023-04-24 stsp err = open_tree(buf2, len2, pack, packidx, idx, &id2,
585 cdfe5501 2023-04-24 stsp memset(&pte2, 0, sizeof(pte2));
588 cdfe5501 2023-04-24 stsp next_entry2 = *buf2;
589 cdfe5501 2023-04-24 stsp remain2 = *len2;
593 ca6e02ac 2020-01-07 stsp return err;
596 ca6e02ac 2020-01-07 stsp static const struct got_error *
597 e70bf110 2020-03-22 stsp send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
598 e70bf110 2020-03-22 stsp struct imsgbuf *ibuf)
600 e70bf110 2020-03-22 stsp struct ibuf *wbuf;
603 e70bf110 2020-03-22 stsp wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
604 e70bf110 2020-03-22 stsp sizeof(struct got_imsg_traversed_commits) +
605 076fbedc 2023-02-19 op ncommits * sizeof(commit_ids[0]));
606 e70bf110 2020-03-22 stsp if (wbuf == NULL)
607 e70bf110 2020-03-22 stsp return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
609 1453347d 2022-05-19 stsp if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
610 1453347d 2022-05-19 stsp return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
612 e70bf110 2020-03-22 stsp for (i = 0; i < ncommits; i++) {
613 e70bf110 2020-03-22 stsp struct got_object_id *id = &commit_ids[i];
614 076fbedc 2023-02-19 op if (imsg_add(wbuf, id, sizeof(*id)) == -1) {
615 1453347d 2022-05-19 stsp return got_error_from_errno(
616 e70bf110 2020-03-22 stsp "imsg_add TRAVERSED_COMMITS");
620 e70bf110 2020-03-22 stsp imsg_close(ibuf, wbuf);
622 e70bf110 2020-03-22 stsp return got_privsep_flush_imsg(ibuf);
625 e70bf110 2020-03-22 stsp static const struct got_error *
626 e70bf110 2020-03-22 stsp send_commit_traversal_done(struct imsgbuf *ibuf)
628 e70bf110 2020-03-22 stsp if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
629 e70bf110 2020-03-22 stsp NULL, 0) == -1)
630 e70bf110 2020-03-22 stsp return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
632 e70bf110 2020-03-22 stsp return got_privsep_flush_imsg(ibuf);
635 e70bf110 2020-03-22 stsp static const struct got_error *
636 ca6e02ac 2020-01-07 stsp commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
637 ca6e02ac 2020-01-07 stsp struct got_pack *pack, struct got_packidx *packidx,
638 ca6e02ac 2020-01-07 stsp struct got_object_cache *objcache)
640 ca6e02ac 2020-01-07 stsp const struct got_error *err = NULL;
641 47cb6f8b 2023-02-26 op struct got_imsg_commit_traversal_request ctreq;
642 ca6e02ac 2020-01-07 stsp struct got_object_qid *pid;
643 ca6e02ac 2020-01-07 stsp struct got_commit_object *commit = NULL, *pcommit = NULL;
644 ca6e02ac 2020-01-07 stsp struct got_object_id id;
645 47cb6f8b 2023-02-26 op size_t datalen;
646 ca6e02ac 2020-01-07 stsp char *path = NULL;
647 ca6e02ac 2020-01-07 stsp const int min_alloc = 64;
648 ca6e02ac 2020-01-07 stsp int changed = 0, ncommits = 0, nallocated = 0;
649 ca6e02ac 2020-01-07 stsp struct got_object_id *commit_ids = NULL;
651 ca6e02ac 2020-01-07 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
652 47cb6f8b 2023-02-26 op if (datalen < sizeof(ctreq))
653 47cb6f8b 2023-02-26 op return got_error(GOT_ERR_PRIVSEP_LEN);
654 47cb6f8b 2023-02-26 op memcpy(&ctreq, imsg->data, sizeof(ctreq));
655 47cb6f8b 2023-02-26 op memcpy(&id, &ctreq.iobj.id, sizeof(id));
657 47cb6f8b 2023-02-26 op if (datalen != sizeof(ctreq) + ctreq.path_len)
658 ca6e02ac 2020-01-07 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
659 47cb6f8b 2023-02-26 op if (ctreq.path_len == 0)
660 47cb6f8b 2023-02-26 op return got_error(GOT_ERR_PRIVSEP_LEN);
662 47cb6f8b 2023-02-26 op path = strndup(imsg->data + sizeof(ctreq), ctreq.path_len);
663 47cb6f8b 2023-02-26 op if (path == NULL)
664 47cb6f8b 2023-02-26 op return got_error_from_errno("strndup");
666 ca6e02ac 2020-01-07 stsp nallocated = min_alloc;
667 ca6e02ac 2020-01-07 stsp commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
668 ca6e02ac 2020-01-07 stsp if (commit_ids == NULL)
669 ca6e02ac 2020-01-07 stsp return got_error_from_errno("reallocarray");
672 ca6e02ac 2020-01-07 stsp const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
675 ca6e02ac 2020-01-07 stsp if (sigint_received) {
676 ca6e02ac 2020-01-07 stsp err = got_error(GOT_ERR_CANCELLED);
680 ca6e02ac 2020-01-07 stsp if (commit == NULL) {
681 ca6e02ac 2020-01-07 stsp idx = got_packidx_get_object_idx(packidx, &id);
682 ca6e02ac 2020-01-07 stsp if (idx == -1)
684 ca6e02ac 2020-01-07 stsp err = open_commit(&commit, pack, packidx,
685 ca6e02ac 2020-01-07 stsp idx, &id, objcache);
687 ca6e02ac 2020-01-07 stsp if (err->code != GOT_ERR_NO_OBJ)
689 ca6e02ac 2020-01-07 stsp err = NULL;
694 ca6e02ac 2020-01-07 stsp if (sizeof(struct got_imsg_traversed_commits) +
695 076fbedc 2023-02-19 op ncommits * sizeof(commit_ids[0]) >= max_datalen) {
696 e70bf110 2020-03-22 stsp err = send_traversed_commits(commit_ids, ncommits,
700 ca6e02ac 2020-01-07 stsp ncommits = 0;
702 ca6e02ac 2020-01-07 stsp ncommits++;
703 ca6e02ac 2020-01-07 stsp if (ncommits > nallocated) {
704 ca6e02ac 2020-01-07 stsp struct got_object_id *new;
705 ca6e02ac 2020-01-07 stsp nallocated += min_alloc;
706 ca6e02ac 2020-01-07 stsp new = reallocarray(commit_ids, nallocated,
707 ca6e02ac 2020-01-07 stsp sizeof(*commit_ids));
708 ca6e02ac 2020-01-07 stsp if (new == NULL) {
709 ca6e02ac 2020-01-07 stsp err = got_error_from_errno("reallocarray");
712 ca6e02ac 2020-01-07 stsp commit_ids = new;
714 b685c8da 2023-02-19 op memcpy(&commit_ids[ncommits - 1], &id, sizeof(id));
716 dbdddfee 2021-06-23 naddy pid = STAILQ_FIRST(&commit->parent_ids);
717 ca6e02ac 2020-01-07 stsp if (pid == NULL)
720 d7b5a0e8 2022-04-20 stsp idx = got_packidx_get_object_idx(packidx, &pid->id);
721 ca6e02ac 2020-01-07 stsp if (idx == -1)
724 d7b5a0e8 2022-04-20 stsp err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
727 ca6e02ac 2020-01-07 stsp if (err->code != GOT_ERR_NO_OBJ)
729 ca6e02ac 2020-01-07 stsp err = NULL;
733 ca6e02ac 2020-01-07 stsp if (path[0] == '/' && path[1] == '\0') {
734 ca6e02ac 2020-01-07 stsp if (got_object_id_cmp(pcommit->tree_id,
735 ca6e02ac 2020-01-07 stsp commit->tree_id) != 0) {
736 ca6e02ac 2020-01-07 stsp changed = 1;
741 ca6e02ac 2020-01-07 stsp uint8_t *buf = NULL, *pbuf = NULL;
742 cdfe5501 2023-04-24 stsp size_t len = 0, plen = 0;
744 ca6e02ac 2020-01-07 stsp idx = got_packidx_get_object_idx(packidx,
745 ca6e02ac 2020-01-07 stsp commit->tree_id);
746 ca6e02ac 2020-01-07 stsp if (idx == -1)
748 ca6e02ac 2020-01-07 stsp pidx = got_packidx_get_object_idx(packidx,
749 ca6e02ac 2020-01-07 stsp pcommit->tree_id);
750 ca6e02ac 2020-01-07 stsp if (pidx == -1)
753 cdfe5501 2023-04-24 stsp err = open_tree(&buf, &len, pack, packidx, idx,
754 d294b1dc 2022-10-18 stsp commit->tree_id, objcache);
758 cdfe5501 2023-04-24 stsp err = open_tree(&pbuf, &plen, pack, packidx, pidx,
759 d294b1dc 2022-10-18 stsp pcommit->tree_id, objcache);
765 cdfe5501 2023-04-24 stsp err = tree_path_changed(&changed, &buf, &len,
766 cdfe5501 2023-04-24 stsp &pbuf, &plen, path, pack, packidx, ibuf,
770 ca6e02ac 2020-01-07 stsp free(pbuf);
772 ca6e02ac 2020-01-07 stsp if (err->code != GOT_ERR_NO_OBJ)
774 ca6e02ac 2020-01-07 stsp err = NULL;
779 ca6e02ac 2020-01-07 stsp if (!changed) {
780 b685c8da 2023-02-19 op memcpy(&id, &pid->id, sizeof(id));
781 ca6e02ac 2020-01-07 stsp got_object_commit_close(commit);
782 ca6e02ac 2020-01-07 stsp commit = pcommit;
783 ca6e02ac 2020-01-07 stsp pcommit = NULL;
785 ca6e02ac 2020-01-07 stsp } while (!changed);
787 ca6e02ac 2020-01-07 stsp if (ncommits > 0) {
788 e70bf110 2020-03-22 stsp err = send_traversed_commits(commit_ids, ncommits, ibuf);
792 ca6e02ac 2020-01-07 stsp if (changed) {
793 ca6e02ac 2020-01-07 stsp err = got_privsep_send_commit(ibuf, commit);
798 e70bf110 2020-03-22 stsp err = send_commit_traversal_done(ibuf);
801 ca6e02ac 2020-01-07 stsp free(commit_ids);
802 ca6e02ac 2020-01-07 stsp if (commit)
803 ca6e02ac 2020-01-07 stsp got_object_commit_close(commit);
804 ca6e02ac 2020-01-07 stsp if (pcommit)
805 ca6e02ac 2020-01-07 stsp got_object_commit_close(pcommit);
807 f4a881ce 2018-11-17 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
808 f4a881ce 2018-11-17 stsp err = NULL;
810 f4a881ce 2018-11-17 stsp got_privsep_send_error(ibuf, err);
813 f4a881ce 2018-11-17 stsp return err;
816 f4a881ce 2018-11-17 stsp static const struct got_error *
817 c0df5966 2021-12-31 stsp raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
818 c0df5966 2021-12-31 stsp struct got_pack *pack, struct got_packidx *packidx,
819 db696021 2022-01-04 stsp struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
821 59d1e4a0 2021-03-10 stsp const struct got_error *err = NULL;
822 59d1e4a0 2021-03-10 stsp uint8_t *buf = NULL;
823 59d1e4a0 2021-03-10 stsp uint64_t size = 0;
824 db696021 2022-01-04 stsp FILE *outfile = NULL;
825 59d1e4a0 2021-03-10 stsp struct got_imsg_packed_object iobj;
826 59d1e4a0 2021-03-10 stsp struct got_object *obj;
827 59d1e4a0 2021-03-10 stsp struct got_object_id id;
828 59d1e4a0 2021-03-10 stsp size_t datalen;
830 59d1e4a0 2021-03-10 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
831 59d1e4a0 2021-03-10 stsp if (datalen != sizeof(iobj))
832 59d1e4a0 2021-03-10 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
833 59d1e4a0 2021-03-10 stsp memcpy(&iobj, imsg->data, sizeof(iobj));
834 265df21f 2023-01-31 op memcpy(&id, &iobj.id, sizeof(id));
836 59d1e4a0 2021-03-10 stsp obj = got_object_cache_get(objcache, &id);
838 59d1e4a0 2021-03-10 stsp obj->refcnt++;
840 59d1e4a0 2021-03-10 stsp err = open_object(&obj, pack, packidx, iobj.idx, &id,
843 59d1e4a0 2021-03-10 stsp return err;
846 59d1e4a0 2021-03-10 stsp err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
848 59d1e4a0 2021-03-10 stsp return err;
850 59d1e4a0 2021-03-10 stsp if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
851 59d1e4a0 2021-03-10 stsp err = got_pack_get_max_delta_object_size(&size, obj, pack);
855 59d1e4a0 2021-03-10 stsp size = obj->size;
857 59d1e4a0 2021-03-10 stsp if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
858 59d1e4a0 2021-03-10 stsp err = got_packfile_extract_object_to_mem(&buf, &obj->size,
859 59d1e4a0 2021-03-10 stsp obj, pack);
861 59d1e4a0 2021-03-10 stsp err = got_packfile_extract_object(pack, obj, outfile, basefile,
862 59d1e4a0 2021-03-10 stsp accumfile);
866 40e3cb72 2021-06-22 stsp err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
869 59d1e4a0 2021-03-10 stsp if (outfile && fclose(outfile) == EOF && err == NULL)
870 59d1e4a0 2021-03-10 stsp err = got_error_from_errno("fclose");
871 59d1e4a0 2021-03-10 stsp got_object_close(obj);
872 59d1e4a0 2021-03-10 stsp if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
873 59d1e4a0 2021-03-10 stsp got_privsep_send_error(ibuf, err);
875 59d1e4a0 2021-03-10 stsp return err;
878 67fd6849 2022-02-13 stsp static const struct got_error *
879 67fd6849 2022-02-13 stsp get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
880 67fd6849 2022-02-13 stsp off_t base_offset)
882 67fd6849 2022-02-13 stsp const struct got_error *err;
885 5e91dae4 2022-08-30 stsp err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
887 67fd6849 2022-02-13 stsp return err;
888 67fd6849 2022-02-13 stsp if (idx == -1)
889 67fd6849 2022-02-13 stsp return got_error(GOT_ERR_BAD_PACKIDX);
891 67fd6849 2022-02-13 stsp return got_packidx_get_object_id(base_id, packidx, idx);
894 59d1e4a0 2021-03-10 stsp static const struct got_error *
895 67fd6849 2022-02-13 stsp raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
896 67fd6849 2022-02-13 stsp FILE *delta_outfile, struct got_pack *pack,
897 67fd6849 2022-02-13 stsp struct got_packidx *packidx)
899 67fd6849 2022-02-13 stsp const struct got_error *err = NULL;
900 67fd6849 2022-02-13 stsp struct got_imsg_raw_delta_request req;
901 2d9e6abf 2022-05-04 stsp size_t datalen, delta_size, delta_compressed_size;
902 24b7de1c 2022-12-03 stsp off_t delta_offset, delta_data_offset;
903 67fd6849 2022-02-13 stsp uint8_t *delta_buf = NULL;
904 67fd6849 2022-02-13 stsp struct got_object_id id, base_id;
905 67fd6849 2022-02-13 stsp off_t base_offset, delta_out_offset = 0;
906 67fd6849 2022-02-13 stsp uint64_t base_size = 0, result_size = 0;
909 67fd6849 2022-02-13 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
910 67fd6849 2022-02-13 stsp if (datalen != sizeof(req))
911 67fd6849 2022-02-13 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
912 67fd6849 2022-02-13 stsp memcpy(&req, imsg->data, sizeof(req));
913 babd9f5d 2023-01-31 op memcpy(&id, &req.id, sizeof(id));
915 67fd6849 2022-02-13 stsp err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
916 24b7de1c 2022-12-03 stsp &delta_compressed_size, &delta_offset, &delta_data_offset,
917 24b7de1c 2022-12-03 stsp &base_offset, &base_id, &base_size, &result_size,
918 24b7de1c 2022-12-03 stsp pack, packidx, req.idx);
923 67fd6849 2022-02-13 stsp * If this is an offset delta we must determine the base
924 67fd6849 2022-02-13 stsp * object ID ourselves.
926 67fd6849 2022-02-13 stsp if (base_offset != 0) {
927 67fd6849 2022-02-13 stsp err = get_base_object_id(&base_id, packidx, base_offset);
932 67fd6849 2022-02-13 stsp delta_out_offset = ftello(delta_outfile);
933 2d9e6abf 2022-05-04 stsp w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
934 2d9e6abf 2022-05-04 stsp if (w != delta_compressed_size) {
935 67fd6849 2022-02-13 stsp err = got_ferror(delta_outfile, GOT_ERR_IO);
938 67fd6849 2022-02-13 stsp if (fflush(delta_outfile) == -1) {
939 67fd6849 2022-02-13 stsp err = got_error_from_errno("fflush");
943 67fd6849 2022-02-13 stsp err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
944 2d9e6abf 2022-05-04 stsp delta_size, delta_compressed_size, delta_offset, delta_out_offset,
947 67fd6849 2022-02-13 stsp free(delta_buf);
948 67fd6849 2022-02-13 stsp return err;
951 fae7e038 2022-05-07 stsp struct search_deltas_arg {
952 fae7e038 2022-05-07 stsp struct imsgbuf *ibuf;
953 fae7e038 2022-05-07 stsp struct got_packidx *packidx;
954 fae7e038 2022-05-07 stsp struct got_pack *pack;
955 fae7e038 2022-05-07 stsp struct got_object_idset *idset;
956 fae7e038 2022-05-07 stsp struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
957 fae7e038 2022-05-07 stsp size_t ndeltas;
960 67fd6849 2022-02-13 stsp static const struct got_error *
961 fae7e038 2022-05-07 stsp search_delta_for_object(struct got_object_id *id, void *data, void *arg)
963 fae7e038 2022-05-07 stsp const struct got_error *err;
964 fae7e038 2022-05-07 stsp struct search_deltas_arg *a = arg;
965 fae7e038 2022-05-07 stsp int obj_idx;
966 fae7e038 2022-05-07 stsp uint8_t *delta_buf = NULL;
967 fae7e038 2022-05-07 stsp uint64_t base_size, result_size;
968 fae7e038 2022-05-07 stsp size_t delta_size, delta_compressed_size;
969 24b7de1c 2022-12-03 stsp off_t delta_offset, delta_data_offset, base_offset;
970 fae7e038 2022-05-07 stsp struct got_object_id base_id;
972 fae7e038 2022-05-07 stsp if (sigint_received)
973 fae7e038 2022-05-07 stsp return got_error(GOT_ERR_CANCELLED);
975 fae7e038 2022-05-07 stsp obj_idx = got_packidx_get_object_idx(a->packidx, id);
976 fae7e038 2022-05-07 stsp if (obj_idx == -1)
977 fae7e038 2022-05-07 stsp return NULL; /* object not present in our pack file */
979 fae7e038 2022-05-07 stsp err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
980 24b7de1c 2022-12-03 stsp &delta_compressed_size, &delta_offset, &delta_data_offset,
981 24b7de1c 2022-12-03 stsp &base_offset, &base_id, &base_size, &result_size,
982 24b7de1c 2022-12-03 stsp a->pack, a->packidx, obj_idx);
984 fae7e038 2022-05-07 stsp if (err->code == GOT_ERR_OBJ_TYPE)
985 fae7e038 2022-05-07 stsp return NULL; /* object not stored as a delta */
986 fae7e038 2022-05-07 stsp return err;
990 fae7e038 2022-05-07 stsp * If this is an offset delta we must determine the base
991 fae7e038 2022-05-07 stsp * object ID ourselves.
993 fae7e038 2022-05-07 stsp if (base_offset != 0) {
994 fae7e038 2022-05-07 stsp err = get_base_object_id(&base_id, a->packidx, base_offset);
999 fae7e038 2022-05-07 stsp if (got_object_idset_contains(a->idset, &base_id)) {
1000 fae7e038 2022-05-07 stsp struct got_imsg_reused_delta *delta;
1002 fae7e038 2022-05-07 stsp delta = &a->deltas[a->ndeltas++];
1003 fae7e038 2022-05-07 stsp memcpy(&delta->id, id, sizeof(delta->id));
1004 fae7e038 2022-05-07 stsp memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
1005 fae7e038 2022-05-07 stsp delta->base_size = base_size;
1006 fae7e038 2022-05-07 stsp delta->result_size = result_size;
1007 fae7e038 2022-05-07 stsp delta->delta_size = delta_size;
1008 fae7e038 2022-05-07 stsp delta->delta_compressed_size = delta_compressed_size;
1009 24b7de1c 2022-12-03 stsp delta->delta_offset = delta_data_offset;
1011 fae7e038 2022-05-07 stsp if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
1012 fae7e038 2022-05-07 stsp err = got_privsep_send_reused_deltas(a->ibuf,
1013 fae7e038 2022-05-07 stsp a->deltas, a->ndeltas);
1015 fae7e038 2022-05-07 stsp goto done;
1016 fae7e038 2022-05-07 stsp a->ndeltas = 0;
1020 fae7e038 2022-05-07 stsp free(delta_buf);
1021 fae7e038 2022-05-07 stsp return err;
1024 fae7e038 2022-05-07 stsp static const struct got_error *
1025 fae7e038 2022-05-07 stsp recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1027 fae7e038 2022-05-07 stsp const struct got_error *err = NULL;
1028 fae7e038 2022-05-07 stsp int done = 0;
1029 fae7e038 2022-05-07 stsp struct got_object_id *ids;
1030 fae7e038 2022-05-07 stsp size_t nids, i;
1032 fae7e038 2022-05-07 stsp for (;;) {
1033 fae7e038 2022-05-07 stsp err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1034 fae7e038 2022-05-07 stsp if (err || done)
1036 fae7e038 2022-05-07 stsp for (i = 0; i < nids; i++) {
1037 fae7e038 2022-05-07 stsp err = got_object_idset_add(idset, &ids[i], NULL);
1038 fae7e038 2022-05-07 stsp if (err) {
1039 fae7e038 2022-05-07 stsp free(ids);
1040 fae7e038 2022-05-07 stsp return err;
1043 fae7e038 2022-05-07 stsp free(ids);
1046 0ab4c957 2022-06-13 stsp return err;
1049 0ab4c957 2022-06-13 stsp static const struct got_error *
1050 0a618912 2023-01-27 stsp recv_object_id_queue(struct got_object_id_queue *queue,
1051 0a618912 2023-01-27 stsp struct got_object_idset *queued_ids, struct imsgbuf *ibuf)
1053 0ab4c957 2022-06-13 stsp const struct got_error *err = NULL;
1054 0ab4c957 2022-06-13 stsp int done = 0;
1055 0ab4c957 2022-06-13 stsp struct got_object_qid *qid;
1056 0ab4c957 2022-06-13 stsp struct got_object_id *ids;
1057 0ab4c957 2022-06-13 stsp size_t nids, i;
1059 0ab4c957 2022-06-13 stsp for (;;) {
1060 0ab4c957 2022-06-13 stsp err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1061 0ab4c957 2022-06-13 stsp if (err || done)
1063 0ab4c957 2022-06-13 stsp for (i = 0; i < nids; i++) {
1064 0ab4c957 2022-06-13 stsp err = got_object_qid_alloc_partial(&qid);
1066 0ab4c957 2022-06-13 stsp return err;
1067 0ab4c957 2022-06-13 stsp memcpy(&qid->id, &ids[i], sizeof(qid->id));
1068 0ab4c957 2022-06-13 stsp STAILQ_INSERT_TAIL(queue, qid, entry);
1069 0a618912 2023-01-27 stsp err = got_object_idset_add(queued_ids, &qid->id, NULL);
1071 0a618912 2023-01-27 stsp return err;
1075 cee6a7ea 2022-06-07 stsp return err;
1078 cee6a7ea 2022-06-07 stsp static const struct got_error *
1079 fae7e038 2022-05-07 stsp delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1080 24b7de1c 2022-12-03 stsp struct got_pack *pack, struct got_packidx *packidx)
1082 fae7e038 2022-05-07 stsp const struct got_error *err = NULL;
1083 fae7e038 2022-05-07 stsp struct got_object_idset *idset;
1084 fae7e038 2022-05-07 stsp struct search_deltas_arg sda;
1086 fae7e038 2022-05-07 stsp idset = got_object_idset_alloc();
1087 fae7e038 2022-05-07 stsp if (idset == NULL)
1088 fae7e038 2022-05-07 stsp return got_error_from_errno("got_object_idset_alloc");
1090 fae7e038 2022-05-07 stsp err = recv_object_ids(idset, ibuf);
1092 fae7e038 2022-05-07 stsp return err;
1094 fae7e038 2022-05-07 stsp memset(&sda, 0, sizeof(sda));
1095 fae7e038 2022-05-07 stsp sda.ibuf = ibuf;
1096 fae7e038 2022-05-07 stsp sda.idset = idset;
1097 fae7e038 2022-05-07 stsp sda.pack = pack;
1098 fae7e038 2022-05-07 stsp sda.packidx = packidx;
1099 fae7e038 2022-05-07 stsp err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1101 fae7e038 2022-05-07 stsp goto done;
1103 fae7e038 2022-05-07 stsp if (sda.ndeltas > 0) {
1104 fae7e038 2022-05-07 stsp err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1105 fae7e038 2022-05-07 stsp sda.ndeltas);
1107 fae7e038 2022-05-07 stsp goto done;
1110 fae7e038 2022-05-07 stsp err = got_privsep_send_reused_deltas_done(ibuf);
1112 fae7e038 2022-05-07 stsp got_object_idset_free(idset);
1113 fae7e038 2022-05-07 stsp return err;
1116 fae7e038 2022-05-07 stsp static const struct got_error *
1117 876c234b 2018-09-10 stsp receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1119 876c234b 2018-09-10 stsp const struct got_error *err = NULL;
1120 876c234b 2018-09-10 stsp struct imsg imsg;
1121 876c234b 2018-09-10 stsp struct got_imsg_packidx ipackidx;
1122 876c234b 2018-09-10 stsp size_t datalen;
1123 876c234b 2018-09-10 stsp struct got_packidx *p;
1125 876c234b 2018-09-10 stsp *packidx = NULL;
1127 876c234b 2018-09-10 stsp err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1129 876c234b 2018-09-10 stsp return err;
1131 876c234b 2018-09-10 stsp p = calloc(1, sizeof(*p));
1132 876c234b 2018-09-10 stsp if (p == NULL) {
1133 638f9024 2019-05-13 stsp err = got_error_from_errno("calloc");
1134 876c234b 2018-09-10 stsp goto done;
1137 876c234b 2018-09-10 stsp if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1138 876c234b 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1139 876c234b 2018-09-10 stsp goto done;
1142 876c234b 2018-09-10 stsp datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1143 876c234b 2018-09-10 stsp if (datalen != sizeof(ipackidx)) {
1144 876c234b 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
1145 876c234b 2018-09-10 stsp goto done;
1147 876c234b 2018-09-10 stsp memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1149 2c52c623 2024-01-30 op p->fd = imsg_get_fd(&imsg);
1150 876c234b 2018-09-10 stsp p->len = ipackidx.len;
1151 876c234b 2018-09-10 stsp if (p->fd == -1) {
1152 2c52c623 2024-01-30 op err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1153 56bef47a 2018-09-15 stsp goto done;
1155 56bef47a 2018-09-15 stsp if (lseek(p->fd, 0, SEEK_SET) == -1) {
1156 638f9024 2019-05-13 stsp err = got_error_from_errno("lseek");
1157 876c234b 2018-09-10 stsp goto done;
1160 876c234b 2018-09-10 stsp #ifndef GOT_PACK_NO_MMAP
1161 1c28a361 2022-10-25 op if (p->len > 0 && p->len <= SIZE_MAX) {
1162 1c28a361 2022-10-25 op p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1163 1c28a361 2022-10-25 op if (p->map == MAP_FAILED)
1164 1c28a361 2022-10-25 op p->map = NULL; /* fall back to read(2) */
1167 c3564dfa 2021-07-15 stsp err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1169 876c234b 2018-09-10 stsp if (err) {
1170 2c52c623 2024-01-30 op if (p != NULL)
1171 2c52c623 2024-01-30 op got_packidx_close(p);
1173 876c234b 2018-09-10 stsp *packidx = p;
1174 876c234b 2018-09-10 stsp imsg_free(&imsg);
1175 0ab4c957 2022-06-13 stsp return err;
1178 0ab4c957 2022-06-13 stsp static const struct got_error *
1179 0ab4c957 2022-06-13 stsp send_tree_enumeration_done(struct imsgbuf *ibuf)
1181 0ab4c957 2022-06-13 stsp if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1182 0ab4c957 2022-06-13 stsp NULL, 0) == -1)
1183 0ab4c957 2022-06-13 stsp return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1185 0ab4c957 2022-06-13 stsp return got_privsep_flush_imsg(ibuf);
1188 0ab4c957 2022-06-13 stsp struct enumerated_tree {
1189 0ab4c957 2022-06-13 stsp struct got_object_id id;
1190 0ab4c957 2022-06-13 stsp char *path;
1191 0ab4c957 2022-06-13 stsp uint8_t *buf;
1192 0ab4c957 2022-06-13 stsp struct got_parsed_tree_entry *entries;
1193 0ab4c957 2022-06-13 stsp int nentries;
1196 0ab4c957 2022-06-13 stsp static const struct got_error *
1197 0ab4c957 2022-06-13 stsp enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1198 0ab4c957 2022-06-13 stsp struct got_object_id *tree_id,
1199 0ab4c957 2022-06-13 stsp const char *path, struct got_pack *pack, struct got_packidx *packidx,
1200 0ab4c957 2022-06-13 stsp struct got_object_cache *objcache, struct got_object_idset *idset,
1201 0ab4c957 2022-06-13 stsp struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1203 0ab4c957 2022-06-13 stsp const struct got_error *err = NULL;
1204 0ab4c957 2022-06-13 stsp struct got_object_id_queue ids;
1205 0ab4c957 2022-06-13 stsp struct got_object_qid *qid;
1206 0ab4c957 2022-06-13 stsp uint8_t *buf = NULL;
1207 cdfe5501 2023-04-24 stsp size_t len = 0;
1208 0ab4c957 2022-06-13 stsp struct got_parsed_tree_entry *entries = NULL;
1209 d294b1dc 2022-10-18 stsp size_t nentries = 0, nentries_alloc = 0, i;
1210 0ab4c957 2022-06-13 stsp struct enumerated_tree *tree;
1212 0ab4c957 2022-06-13 stsp *ntrees = 0;
1213 0ab4c957 2022-06-13 stsp *have_all_entries = 1;
1214 0ab4c957 2022-06-13 stsp STAILQ_INIT(&ids);
1216 0ab4c957 2022-06-13 stsp err = got_object_qid_alloc_partial(&qid);
1218 0ab4c957 2022-06-13 stsp return err;
1219 076fbedc 2023-02-19 op memcpy(&qid->id, tree_id, sizeof(*tree_id));
1220 0ab4c957 2022-06-13 stsp qid->data = strdup(path);
1221 0ab4c957 2022-06-13 stsp if (qid->data == NULL) {
1222 0ab4c957 2022-06-13 stsp err = got_error_from_errno("strdup");
1223 0ab4c957 2022-06-13 stsp goto done;
1225 0ab4c957 2022-06-13 stsp STAILQ_INSERT_TAIL(&ids, qid, entry);
1226 0ab4c957 2022-06-13 stsp qid = NULL;
1228 0ab4c957 2022-06-13 stsp /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1230 0ab4c957 2022-06-13 stsp const char *path;
1231 0ab4c957 2022-06-13 stsp int idx, i;
1233 0ab4c957 2022-06-13 stsp if (sigint_received) {
1234 0ab4c957 2022-06-13 stsp err = got_error(GOT_ERR_CANCELLED);
1235 0ab4c957 2022-06-13 stsp goto done;
1238 0ab4c957 2022-06-13 stsp qid = STAILQ_FIRST(&ids);
1239 0ab4c957 2022-06-13 stsp STAILQ_REMOVE_HEAD(&ids, entry);
1240 0ab4c957 2022-06-13 stsp path = qid->data;
1242 0ab4c957 2022-06-13 stsp idx = got_packidx_get_object_idx(packidx, &qid->id);
1243 0ab4c957 2022-06-13 stsp if (idx == -1) {
1244 0ab4c957 2022-06-13 stsp *have_all_entries = 0;
1248 cdfe5501 2023-04-24 stsp err = open_tree(&buf, &len, pack, packidx, idx, &qid->id,
1249 cdfe5501 2023-04-24 stsp objcache);
1250 0ab4c957 2022-06-13 stsp if (err) {
1251 0ab4c957 2022-06-13 stsp if (err->code != GOT_ERR_NO_OBJ)
1252 0ab4c957 2022-06-13 stsp goto done;
1255 cdfe5501 2023-04-24 stsp err = got_object_parse_tree(&entries, &nentries,
1256 cdfe5501 2023-04-24 stsp &nentries_alloc, buf, len);
1258 cdfe5501 2023-04-24 stsp goto done;
1260 0ab4c957 2022-06-13 stsp err = got_object_idset_add(idset, &qid->id, NULL);
1262 0ab4c957 2022-06-13 stsp goto done;
1264 0ab4c957 2022-06-13 stsp for (i = 0; i < nentries; i++) {
1265 0ab4c957 2022-06-13 stsp struct got_object_qid *eqid = NULL;
1266 0ab4c957 2022-06-13 stsp struct got_parsed_tree_entry *pte = &entries[i];
1269 0ab4c957 2022-06-13 stsp if (!S_ISDIR(pte->mode))
1272 0ab4c957 2022-06-13 stsp err = got_object_qid_alloc_partial(&eqid);
1274 0ab4c957 2022-06-13 stsp goto done;
1275 0ab4c957 2022-06-13 stsp memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1277 0ab4c957 2022-06-13 stsp if (got_object_idset_contains(idset, &eqid->id)) {
1278 0ab4c957 2022-06-13 stsp got_object_qid_free(eqid);
1282 0ab4c957 2022-06-13 stsp if (asprintf(&p, "%s%s%s", path,
1283 0ab4c957 2022-06-13 stsp got_path_is_root_dir(path) ? "" : "/",
1284 0ab4c957 2022-06-13 stsp pte->name) == -1) {
1285 0ab4c957 2022-06-13 stsp err = got_error_from_errno("asprintf");
1286 0ab4c957 2022-06-13 stsp got_object_qid_free(eqid);
1287 0ab4c957 2022-06-13 stsp goto done;
1289 0ab4c957 2022-06-13 stsp eqid->data = p;
1290 0ab4c957 2022-06-13 stsp STAILQ_INSERT_TAIL(&ids, eqid, entry);
1293 0ab4c957 2022-06-13 stsp if (*ntrees >= *nalloc) {
1294 0ab4c957 2022-06-13 stsp struct enumerated_tree *new;
1295 0ab4c957 2022-06-13 stsp new = recallocarray(*trees, *nalloc, *nalloc + 16,
1296 0ab4c957 2022-06-13 stsp sizeof(*new));
1297 0ab4c957 2022-06-13 stsp if (new == NULL) {
1298 0ab4c957 2022-06-13 stsp err = got_error_from_errno("malloc");
1299 0ab4c957 2022-06-13 stsp goto done;
1301 0ab4c957 2022-06-13 stsp *trees = new;
1302 0ab4c957 2022-06-13 stsp *nalloc += 16;
1304 0ab4c957 2022-06-13 stsp tree = &(*trees)[*ntrees];
1305 0ab4c957 2022-06-13 stsp (*ntrees)++;
1306 0ab4c957 2022-06-13 stsp memcpy(&tree->id, &qid->id, sizeof(tree->id));
1307 0ab4c957 2022-06-13 stsp tree->path = qid->data;
1308 0ab4c957 2022-06-13 stsp tree->buf = buf;
1309 0ab4c957 2022-06-13 stsp buf = NULL;
1310 0ab4c957 2022-06-13 stsp tree->entries = entries;
1311 0ab4c957 2022-06-13 stsp entries = NULL;
1312 d294b1dc 2022-10-18 stsp nentries_alloc = 0;
1313 0ab4c957 2022-06-13 stsp tree->nentries = nentries;
1314 d294b1dc 2022-10-18 stsp nentries = 0;
1316 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1317 0ab4c957 2022-06-13 stsp qid = NULL;
1318 0ab4c957 2022-06-13 stsp } while (!STAILQ_EMPTY(&ids));
1320 0ab4c957 2022-06-13 stsp if (*have_all_entries) {
1323 0ab4c957 2022-06-13 stsp * We have managed to traverse all entries in the hierarchy.
1324 0ab4c957 2022-06-13 stsp * Tell the main process what we have found.
1326 0ab4c957 2022-06-13 stsp for (i = 0; i < *ntrees; i++) {
1327 0ab4c957 2022-06-13 stsp tree = &(*trees)[i];
1328 0ab4c957 2022-06-13 stsp err = got_privsep_send_enumerated_tree(totlen,
1329 0ab4c957 2022-06-13 stsp ibuf, &tree->id, tree->path, tree->entries,
1330 0ab4c957 2022-06-13 stsp tree->nentries);
1332 0ab4c957 2022-06-13 stsp goto done;
1333 0ab4c957 2022-06-13 stsp free(tree->buf);
1334 0ab4c957 2022-06-13 stsp tree->buf = NULL;
1335 0ab4c957 2022-06-13 stsp free(tree->path);
1336 0ab4c957 2022-06-13 stsp tree->path = NULL;
1337 0ab4c957 2022-06-13 stsp free(tree->entries);
1338 0ab4c957 2022-06-13 stsp tree->entries = NULL;
1340 0ab4c957 2022-06-13 stsp *ntrees = 0; /* don't loop again below to free memory */
1342 0ab4c957 2022-06-13 stsp err = send_tree_enumeration_done(ibuf);
1345 0ab4c957 2022-06-13 stsp * We can only load fully packed tree hierarchies on
1346 0ab4c957 2022-06-13 stsp * behalf of the main process, otherwise the main process
1347 0ab4c957 2022-06-13 stsp * gets a wrong idea about which tree objects have
1348 0ab4c957 2022-06-13 stsp * already been traversed.
1349 0ab4c957 2022-06-13 stsp * Indicate a missing entry for the root of this tree.
1350 0ab4c957 2022-06-13 stsp * The main process should continue by loading this
1351 0ab4c957 2022-06-13 stsp * entire tree the slow way.
1353 0ab4c957 2022-06-13 stsp err = got_privsep_send_enumerated_tree(totlen, ibuf,
1354 0ab4c957 2022-06-13 stsp tree_id, "/", NULL, -1);
1356 0ab4c957 2022-06-13 stsp goto done;
1359 0ab4c957 2022-06-13 stsp free(buf);
1360 0ab4c957 2022-06-13 stsp free(entries);
1361 0ab4c957 2022-06-13 stsp for (i = 0; i < *ntrees; i++) {
1362 0ab4c957 2022-06-13 stsp tree = &(*trees)[i];
1363 0ab4c957 2022-06-13 stsp free(tree->buf);
1364 0ab4c957 2022-06-13 stsp tree->buf = NULL;
1365 0ab4c957 2022-06-13 stsp free(tree->path);
1366 0ab4c957 2022-06-13 stsp tree->path = NULL;
1367 0ab4c957 2022-06-13 stsp free(tree->entries);
1368 0ab4c957 2022-06-13 stsp tree->entries = NULL;
1371 0ab4c957 2022-06-13 stsp free(qid->data);
1372 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1373 0ab4c957 2022-06-13 stsp got_object_id_queue_free(&ids);
1374 0ab4c957 2022-06-13 stsp if (err) {
1375 0ab4c957 2022-06-13 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
1376 0ab4c957 2022-06-13 stsp err = NULL;
1378 0ab4c957 2022-06-13 stsp got_privsep_send_error(ibuf, err);
1381 cee6a7ea 2022-06-07 stsp return err;
1384 cee6a7ea 2022-06-07 stsp static const struct got_error *
1385 0ab4c957 2022-06-13 stsp enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1386 0ab4c957 2022-06-13 stsp struct got_pack *pack, struct got_packidx *packidx,
1387 0ab4c957 2022-06-13 stsp struct got_object_cache *objcache)
1389 0ab4c957 2022-06-13 stsp const struct got_error *err = NULL;
1390 0ab4c957 2022-06-13 stsp struct got_object_id_queue commit_ids;
1391 0ab4c957 2022-06-13 stsp const struct got_object_id_queue *parents = NULL;
1392 0ab4c957 2022-06-13 stsp struct got_object_qid *qid = NULL;
1393 0ab4c957 2022-06-13 stsp struct got_object *obj = NULL;
1394 0ab4c957 2022-06-13 stsp struct got_commit_object *commit = NULL;
1395 0ab4c957 2022-06-13 stsp struct got_object_id *tree_id = NULL;
1396 0ab4c957 2022-06-13 stsp size_t totlen = 0;
1397 0a618912 2023-01-27 stsp struct got_object_idset *idset, *queued_ids = NULL;
1398 0ab4c957 2022-06-13 stsp int i, idx, have_all_entries = 1;
1399 0ab4c957 2022-06-13 stsp struct enumerated_tree *trees = NULL;
1400 0ab4c957 2022-06-13 stsp size_t ntrees = 0, nalloc = 16;
1402 0ab4c957 2022-06-13 stsp STAILQ_INIT(&commit_ids);
1404 ffe3518f 2022-06-14 stsp trees = calloc(nalloc, sizeof(*trees));
1405 0ab4c957 2022-06-13 stsp if (trees == NULL)
1406 0ab4c957 2022-06-13 stsp return got_error_from_errno("calloc");
1408 0ab4c957 2022-06-13 stsp idset = got_object_idset_alloc();
1409 0ab4c957 2022-06-13 stsp if (idset == NULL) {
1410 0ab4c957 2022-06-13 stsp err = got_error_from_errno("got_object_idset_alloc");
1411 0ab4c957 2022-06-13 stsp goto done;
1414 0a618912 2023-01-27 stsp queued_ids = got_object_idset_alloc();
1415 0a618912 2023-01-27 stsp if (queued_ids == NULL) {
1416 0a618912 2023-01-27 stsp err = got_error_from_errno("got_object_idset_alloc");
1417 0a618912 2023-01-27 stsp goto done;
1420 0a618912 2023-01-27 stsp err = recv_object_id_queue(&commit_ids, queued_ids, ibuf);
1422 0ab4c957 2022-06-13 stsp goto done;
1424 a5e587e0 2022-06-14 stsp if (STAILQ_EMPTY(&commit_ids)) {
1425 a5e587e0 2022-06-14 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1426 a5e587e0 2022-06-14 stsp goto done;
1429 0ab4c957 2022-06-13 stsp err = recv_object_ids(idset, ibuf);
1431 0ab4c957 2022-06-13 stsp goto done;
1433 0ab4c957 2022-06-13 stsp while (!STAILQ_EMPTY(&commit_ids)) {
1434 0ab4c957 2022-06-13 stsp if (sigint_received) {
1435 0ab4c957 2022-06-13 stsp err = got_error(GOT_ERR_CANCELLED);
1436 0ab4c957 2022-06-13 stsp goto done;
1439 0ab4c957 2022-06-13 stsp qid = STAILQ_FIRST(&commit_ids);
1440 0ab4c957 2022-06-13 stsp STAILQ_REMOVE_HEAD(&commit_ids, entry);
1442 0ab4c957 2022-06-13 stsp if (got_object_idset_contains(idset, &qid->id)) {
1443 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1444 0ab4c957 2022-06-13 stsp qid = NULL;
1448 0ab4c957 2022-06-13 stsp idx = got_packidx_get_object_idx(packidx, &qid->id);
1449 db9b9b1c 2022-06-14 stsp if (idx == -1) {
1450 db9b9b1c 2022-06-14 stsp have_all_entries = 0;
1454 0ab4c957 2022-06-13 stsp err = open_object(&obj, pack, packidx, idx, &qid->id,
1455 0ab4c957 2022-06-13 stsp objcache);
1457 0ab4c957 2022-06-13 stsp goto done;
1458 0ab4c957 2022-06-13 stsp if (obj->type == GOT_OBJ_TYPE_TAG) {
1459 0ab4c957 2022-06-13 stsp struct got_tag_object *tag;
1460 0ab4c957 2022-06-13 stsp uint8_t *buf;
1461 0ab4c957 2022-06-13 stsp size_t len;
1462 0ab4c957 2022-06-13 stsp err = got_packfile_extract_object_to_mem(&buf,
1463 0ab4c957 2022-06-13 stsp &len, obj, pack);
1465 0ab4c957 2022-06-13 stsp goto done;
1466 0ab4c957 2022-06-13 stsp obj->size = len;
1467 0ab4c957 2022-06-13 stsp err = got_object_parse_tag(&tag, buf, len);
1468 0ab4c957 2022-06-13 stsp if (err) {
1469 0ab4c957 2022-06-13 stsp free(buf);
1470 0ab4c957 2022-06-13 stsp goto done;
1472 0ab4c957 2022-06-13 stsp idx = got_packidx_get_object_idx(packidx, &tag->id);
1473 db9b9b1c 2022-06-14 stsp if (idx == -1) {
1474 db9b9b1c 2022-06-14 stsp have_all_entries = 0;
1477 0ab4c957 2022-06-13 stsp err = open_commit(&commit, pack, packidx, idx,
1478 0ab4c957 2022-06-13 stsp &tag->id, objcache);
1479 0ab4c957 2022-06-13 stsp got_object_tag_close(tag);
1480 0ab4c957 2022-06-13 stsp free(buf);
1482 0ab4c957 2022-06-13 stsp goto done;
1483 0ab4c957 2022-06-13 stsp } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1484 0ab4c957 2022-06-13 stsp err = open_commit(&commit, pack, packidx, idx,
1485 0ab4c957 2022-06-13 stsp &qid->id, objcache);
1487 0ab4c957 2022-06-13 stsp goto done;
1489 0ab4c957 2022-06-13 stsp err = got_error(GOT_ERR_OBJ_TYPE);
1490 0ab4c957 2022-06-13 stsp goto done;
1492 0ab4c957 2022-06-13 stsp got_object_close(obj);
1493 0ab4c957 2022-06-13 stsp obj = NULL;
1495 0ab4c957 2022-06-13 stsp err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1496 0ab4c957 2022-06-13 stsp got_object_commit_get_committer_time(commit));
1498 0ab4c957 2022-06-13 stsp goto done;
1500 0ab4c957 2022-06-13 stsp tree_id = got_object_commit_get_tree_id(commit);
1501 0ab4c957 2022-06-13 stsp idx = got_packidx_get_object_idx(packidx, tree_id);
1502 0ab4c957 2022-06-13 stsp if (idx == -1) {
1503 db9b9b1c 2022-06-14 stsp have_all_entries = 0;
1504 0ab4c957 2022-06-13 stsp err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1505 0ab4c957 2022-06-13 stsp tree_id, "/", NULL, -1);
1507 0ab4c957 2022-06-13 stsp goto done;
1511 0ab4c957 2022-06-13 stsp if (got_object_idset_contains(idset, tree_id)) {
1512 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1513 0ab4c957 2022-06-13 stsp qid = NULL;
1514 e1380e28 2023-01-27 stsp err = send_tree_enumeration_done(ibuf);
1516 e1380e28 2023-01-27 stsp goto done;
1520 89a34d6e 2022-06-18 stsp err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1521 89a34d6e 2022-06-18 stsp tree_id, "/", pack, packidx, objcache, idset,
1522 89a34d6e 2022-06-18 stsp &trees, &nalloc, &ntrees);
1524 0ab4c957 2022-06-13 stsp goto done;
1526 0ab4c957 2022-06-13 stsp if (!have_all_entries)
1529 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1530 0ab4c957 2022-06-13 stsp qid = NULL;
1532 0ab4c957 2022-06-13 stsp parents = got_object_commit_get_parent_ids(commit);
1533 0ab4c957 2022-06-13 stsp if (parents) {
1534 0ab4c957 2022-06-13 stsp struct got_object_qid *pid;
1535 0ab4c957 2022-06-13 stsp STAILQ_FOREACH(pid, parents, entry) {
1536 0ab4c957 2022-06-13 stsp if (got_object_idset_contains(idset, &pid->id))
1538 0a618912 2023-01-27 stsp if (got_object_idset_contains(queued_ids, &pid->id))
1540 0ab4c957 2022-06-13 stsp err = got_object_qid_alloc_partial(&qid);
1542 0ab4c957 2022-06-13 stsp goto done;
1543 0ab4c957 2022-06-13 stsp memcpy(&qid->id, &pid->id, sizeof(qid->id));
1544 0ab4c957 2022-06-13 stsp STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1545 0ab4c957 2022-06-13 stsp qid = NULL;
1549 0ab4c957 2022-06-13 stsp got_object_commit_close(commit);
1550 0ab4c957 2022-06-13 stsp commit = NULL;
1553 0ab4c957 2022-06-13 stsp if (have_all_entries) {
1554 0ab4c957 2022-06-13 stsp err = got_privsep_send_object_enumeration_done(ibuf);
1556 db9b9b1c 2022-06-14 stsp goto done;
1558 db9b9b1c 2022-06-14 stsp err = got_privsep_send_object_enumeration_incomplete(ibuf);
1560 0ab4c957 2022-06-13 stsp goto done;
1564 0ab4c957 2022-06-13 stsp got_object_close(obj);
1565 0ab4c957 2022-06-13 stsp if (commit)
1566 0ab4c957 2022-06-13 stsp got_object_commit_close(commit);
1567 0ab4c957 2022-06-13 stsp got_object_qid_free(qid);
1568 0ab4c957 2022-06-13 stsp got_object_id_queue_free(&commit_ids);
1569 0ab4c957 2022-06-13 stsp if (idset)
1570 0ab4c957 2022-06-13 stsp got_object_idset_free(idset);
1571 0a618912 2023-01-27 stsp if (queued_ids)
1572 0a618912 2023-01-27 stsp got_object_idset_free(queued_ids);
1573 0ab4c957 2022-06-13 stsp for (i = 0; i < ntrees; i++) {
1574 0ab4c957 2022-06-13 stsp struct enumerated_tree *tree = &trees[i];
1575 0ab4c957 2022-06-13 stsp free(tree->buf);
1576 0ab4c957 2022-06-13 stsp free(tree->path);
1577 0ab4c957 2022-06-13 stsp free(tree->entries);
1579 0ab4c957 2022-06-13 stsp free(trees);
1580 0ab4c957 2022-06-13 stsp return err;
1583 61af9b21 2022-06-28 stsp enum findtwixt_color {
1584 61af9b21 2022-06-28 stsp COLOR_KEEP = 0,
1585 61af9b21 2022-06-28 stsp COLOR_DROP,
1586 61af9b21 2022-06-28 stsp COLOR_SKIP,
1587 61af9b21 2022-06-28 stsp COLOR_MAX,
1590 0ab4c957 2022-06-13 stsp static const struct got_error *
1591 61af9b21 2022-06-28 stsp paint_commit(struct got_object_qid *qid, intptr_t color)
1593 61af9b21 2022-06-28 stsp if (color < 0 || color >= COLOR_MAX)
1594 61af9b21 2022-06-28 stsp return got_error(GOT_ERR_RANGE);
1596 61af9b21 2022-06-28 stsp qid->data = (void *)color;
1597 61af9b21 2022-06-28 stsp return NULL;
1600 61af9b21 2022-06-28 stsp static const struct got_error *
1601 61af9b21 2022-06-28 stsp queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1602 61af9b21 2022-06-28 stsp intptr_t color)
1604 61af9b21 2022-06-28 stsp const struct got_error *err;
1605 61af9b21 2022-06-28 stsp struct got_object_qid *qid;
1607 61af9b21 2022-06-28 stsp err = got_object_qid_alloc_partial(&qid);
1609 61af9b21 2022-06-28 stsp return err;
1611 61af9b21 2022-06-28 stsp memcpy(&qid->id, id, sizeof(qid->id));
1612 61af9b21 2022-06-28 stsp STAILQ_INSERT_TAIL(ids, qid, entry);
1613 61af9b21 2022-06-28 stsp return paint_commit(qid, color);
1616 61af9b21 2022-06-28 stsp static const struct got_error *
1617 61af9b21 2022-06-28 stsp paint_commits(struct got_object_id_queue *ids, int *nids,
1618 61af9b21 2022-06-28 stsp struct got_object_idset *keep, struct got_object_idset *drop,
1619 61af9b21 2022-06-28 stsp struct got_object_idset *skip, struct got_pack *pack,
1620 61af9b21 2022-06-28 stsp struct got_packidx *packidx, struct imsgbuf *ibuf,
1621 61af9b21 2022-06-28 stsp struct got_object_cache *objcache)
1623 61af9b21 2022-06-28 stsp const struct got_error *err = NULL;
1624 61af9b21 2022-06-28 stsp struct got_commit_object *commit = NULL;
1625 61af9b21 2022-06-28 stsp struct got_object_id_queue painted;
1626 61af9b21 2022-06-28 stsp const struct got_object_id_queue *parents;
1627 61af9b21 2022-06-28 stsp struct got_object_qid *qid = NULL;
1628 61af9b21 2022-06-28 stsp int nqueued = *nids, nskip = 0, npainted = 0;
1630 61af9b21 2022-06-28 stsp STAILQ_INIT(&painted);
1632 61af9b21 2022-06-28 stsp while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1634 61af9b21 2022-06-28 stsp intptr_t color;
1636 61af9b21 2022-06-28 stsp if (sigint_received) {
1637 61af9b21 2022-06-28 stsp err = got_error(GOT_ERR_CANCELLED);
1638 61af9b21 2022-06-28 stsp goto done;
1641 61af9b21 2022-06-28 stsp qid = STAILQ_FIRST(ids);
1642 61af9b21 2022-06-28 stsp idx = got_packidx_get_object_idx(packidx, &qid->id);
1643 61af9b21 2022-06-28 stsp if (idx == -1) {
1644 61af9b21 2022-06-28 stsp qid = NULL;
1648 61af9b21 2022-06-28 stsp STAILQ_REMOVE_HEAD(ids, entry);
1649 61af9b21 2022-06-28 stsp nqueued--;
1650 61af9b21 2022-06-28 stsp color = (intptr_t)qid->data;
1651 61af9b21 2022-06-28 stsp if (color == COLOR_SKIP)
1654 61af9b21 2022-06-28 stsp if (got_object_idset_contains(skip, &qid->id)) {
1655 61af9b21 2022-06-28 stsp got_object_qid_free(qid);
1656 61af9b21 2022-06-28 stsp qid = NULL;
1660 61af9b21 2022-06-28 stsp switch (color) {
1661 61af9b21 2022-06-28 stsp case COLOR_KEEP:
1662 61af9b21 2022-06-28 stsp if (got_object_idset_contains(keep, &qid->id)) {
1663 61af9b21 2022-06-28 stsp got_object_qid_free(qid);
1664 61af9b21 2022-06-28 stsp qid = NULL;
1667 61af9b21 2022-06-28 stsp if (got_object_idset_contains(drop, &qid->id)) {
1668 61af9b21 2022-06-28 stsp err = paint_commit(qid, COLOR_SKIP);
1670 61af9b21 2022-06-28 stsp goto done;
1672 61af9b21 2022-06-28 stsp err = got_object_idset_add(keep, &qid->id, NULL);
1674 61af9b21 2022-06-28 stsp goto done;
1676 61af9b21 2022-06-28 stsp case COLOR_DROP:
1677 61af9b21 2022-06-28 stsp if (got_object_idset_contains(drop, &qid->id)) {
1678 61af9b21 2022-06-28 stsp got_object_qid_free(qid);
1679 61af9b21 2022-06-28 stsp qid = NULL;
1682 61af9b21 2022-06-28 stsp if (got_object_idset_contains(keep, &qid->id)) {
1683 61af9b21 2022-06-28 stsp err = paint_commit(qid, COLOR_SKIP);
1685 61af9b21 2022-06-28 stsp goto done;
1687 61af9b21 2022-06-28 stsp err = got_object_idset_add(drop, &qid->id, NULL);
1689 61af9b21 2022-06-28 stsp goto done;
1691 61af9b21 2022-06-28 stsp case COLOR_SKIP:
1692 61af9b21 2022-06-28 stsp if (!got_object_idset_contains(skip, &qid->id)) {
1693 61af9b21 2022-06-28 stsp err = got_object_idset_add(skip, &qid->id,
1696 61af9b21 2022-06-28 stsp goto done;
1700 61af9b21 2022-06-28 stsp /* should not happen */
1701 61af9b21 2022-06-28 stsp err = got_error_fmt(GOT_ERR_NOT_IMPL,
1702 756050ac 2022-07-19 op "%s invalid commit color %"PRIdPTR, __func__,
1704 61af9b21 2022-06-28 stsp goto done;
1707 61af9b21 2022-06-28 stsp err = open_commit(&commit, pack, packidx, idx, &qid->id,
1708 61af9b21 2022-06-28 stsp objcache);
1710 61af9b21 2022-06-28 stsp goto done;
1712 61af9b21 2022-06-28 stsp parents = got_object_commit_get_parent_ids(commit);
1713 61af9b21 2022-06-28 stsp if (parents) {
1714 61af9b21 2022-06-28 stsp struct got_object_qid *pid;
1715 61af9b21 2022-06-28 stsp color = (intptr_t)qid->data;
1716 61af9b21 2022-06-28 stsp STAILQ_FOREACH(pid, parents, entry) {
1717 61af9b21 2022-06-28 stsp err = queue_commit_id(ids, &pid->id, color);
1719 61af9b21 2022-06-28 stsp goto done;
1720 61af9b21 2022-06-28 stsp nqueued++;
1721 61af9b21 2022-06-28 stsp if (color == COLOR_SKIP)
1726 61af9b21 2022-06-28 stsp got_object_commit_close(commit);
1727 61af9b21 2022-06-28 stsp commit = NULL;
1729 61af9b21 2022-06-28 stsp STAILQ_INSERT_TAIL(&painted, qid, entry);
1730 61af9b21 2022-06-28 stsp qid = NULL;
1731 61af9b21 2022-06-28 stsp npainted++;
1733 61af9b21 2022-06-28 stsp err = got_privsep_send_painted_commits(ibuf, &painted,
1734 61af9b21 2022-06-28 stsp &npainted, 1, 0);
1736 61af9b21 2022-06-28 stsp goto done;
1739 61af9b21 2022-06-28 stsp err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1741 61af9b21 2022-06-28 stsp goto done;
1743 61af9b21 2022-06-28 stsp *nids = nqueued;
1745 61af9b21 2022-06-28 stsp if (commit)
1746 61af9b21 2022-06-28 stsp got_object_commit_close(commit);
1747 61af9b21 2022-06-28 stsp got_object_qid_free(qid);
1748 61af9b21 2022-06-28 stsp return err;
1751 61af9b21 2022-06-28 stsp static void
1752 61af9b21 2022-06-28 stsp commit_painting_free(struct got_object_idset **keep,
1753 61af9b21 2022-06-28 stsp struct got_object_idset **drop,
1754 61af9b21 2022-06-28 stsp struct got_object_idset **skip)
1756 61af9b21 2022-06-28 stsp if (*keep) {
1757 61af9b21 2022-06-28 stsp got_object_idset_free(*keep);
1758 61af9b21 2022-06-28 stsp *keep = NULL;
1760 61af9b21 2022-06-28 stsp if (*drop) {
1761 61af9b21 2022-06-28 stsp got_object_idset_free(*drop);
1762 61af9b21 2022-06-28 stsp *drop = NULL;
1764 61af9b21 2022-06-28 stsp if (*skip) {
1765 5e91dae4 2022-08-30 stsp got_object_idset_free(*skip);
1766 61af9b21 2022-06-28 stsp *skip = NULL;
1770 61af9b21 2022-06-28 stsp static const struct got_error *
1771 61af9b21 2022-06-28 stsp commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1772 61af9b21 2022-06-28 stsp struct got_object_idset **drop, struct got_object_idset **skip)
1774 61af9b21 2022-06-28 stsp const struct got_error *err = NULL;
1776 61af9b21 2022-06-28 stsp *keep = got_object_idset_alloc();
1777 61af9b21 2022-06-28 stsp if (*keep == NULL) {
1778 61af9b21 2022-06-28 stsp err = got_error_from_errno("got_object_idset_alloc");
1779 61af9b21 2022-06-28 stsp goto done;
1781 61af9b21 2022-06-28 stsp *drop = got_object_idset_alloc();
1782 61af9b21 2022-06-28 stsp if (*drop == NULL) {
1783 61af9b21 2022-06-28 stsp err = got_error_from_errno("got_object_idset_alloc");
1784 61af9b21 2022-06-28 stsp goto done;
1786 61af9b21 2022-06-28 stsp *skip = got_object_idset_alloc();
1787 61af9b21 2022-06-28 stsp if (*skip == NULL) {
1788 61af9b21 2022-06-28 stsp err = got_error_from_errno("got_object_idset_alloc");
1789 61af9b21 2022-06-28 stsp goto done;
1792 61af9b21 2022-06-28 stsp err = recv_object_ids(*keep, ibuf);
1794 61af9b21 2022-06-28 stsp goto done;
1795 61af9b21 2022-06-28 stsp err = recv_object_ids(*drop, ibuf);
1797 61af9b21 2022-06-28 stsp goto done;
1798 61af9b21 2022-06-28 stsp err = recv_object_ids(*skip, ibuf);
1800 61af9b21 2022-06-28 stsp goto done;
1804 61af9b21 2022-06-28 stsp commit_painting_free(keep, drop, skip);
1806 61af9b21 2022-06-28 stsp return err;
1809 61af9b21 2022-06-28 stsp static const struct got_error *
1810 61af9b21 2022-06-28 stsp commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1811 61af9b21 2022-06-28 stsp struct got_pack *pack, struct got_packidx *packidx,
1812 61af9b21 2022-06-28 stsp struct got_object_cache *objcache, struct got_object_idset *keep,
1813 61af9b21 2022-06-28 stsp struct got_object_idset *drop, struct got_object_idset *skip)
1815 61af9b21 2022-06-28 stsp const struct got_error *err = NULL;
1816 61af9b21 2022-06-28 stsp struct got_imsg_commit_painting_request ireq;
1817 61af9b21 2022-06-28 stsp struct got_object_id id;
1818 61af9b21 2022-06-28 stsp size_t datalen;
1819 61af9b21 2022-06-28 stsp struct got_object_id_queue ids;
1820 61af9b21 2022-06-28 stsp int nids = 0;
1822 61af9b21 2022-06-28 stsp STAILQ_INIT(&ids);
1824 61af9b21 2022-06-28 stsp datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1825 61af9b21 2022-06-28 stsp if (datalen != sizeof(ireq))
1826 61af9b21 2022-06-28 stsp return got_error(GOT_ERR_PRIVSEP_LEN);
1827 61af9b21 2022-06-28 stsp memcpy(&ireq, imsg->data, sizeof(ireq));
1828 076fbedc 2023-02-19 op memcpy(&id, &ireq.id, sizeof(id));
1830 61af9b21 2022-06-28 stsp err = queue_commit_id(&ids, &id, ireq.color);
1832 61af9b21 2022-06-28 stsp return err;
1835 61af9b21 2022-06-28 stsp err = paint_commits(&ids, &nids, keep, drop, skip,
1836 61af9b21 2022-06-28 stsp pack, packidx, ibuf, objcache);
1838 61af9b21 2022-06-28 stsp goto done;
1840 61af9b21 2022-06-28 stsp err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1842 61af9b21 2022-06-28 stsp goto done;
1844 61af9b21 2022-06-28 stsp err = got_privsep_send_painting_commits_done(ibuf);
1846 61af9b21 2022-06-28 stsp got_object_id_queue_free(&ids);
1847 61af9b21 2022-06-28 stsp return err;
1850 61af9b21 2022-06-28 stsp static const struct got_error *
1851 876c234b 2018-09-10 stsp receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1853 876c234b 2018-09-10 stsp const struct got_error *err = NULL;
1854 876c234b 2018-09-10 stsp struct imsg imsg;
1855 876c234b 2018-09-10 stsp struct got_imsg_pack ipack;
1856 876c234b 2018-09-10 stsp size_t datalen;
1857 876c234b 2018-09-10 stsp struct got_pack *pack;
1859 876c234b 2018-09-10 stsp *packp = NULL;
1861 876c234b 2018-09-10 stsp err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1863 876c234b 2018-09-10 stsp return err;
1865 876c234b 2018-09-10 stsp pack = calloc(1, sizeof(*pack));
1866 876c234b 2018-09-10 stsp if (pack == NULL) {
1867 638f9024 2019-05-13 stsp err = got_error_from_errno("calloc");
1868 876c234b 2018-09-10 stsp goto done;
1871 876c234b 2018-09-10 stsp if (imsg.hdr.type != GOT_IMSG_PACK) {
1872 876c234b 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1873 876c234b 2018-09-10 stsp goto done;
1876 876c234b 2018-09-10 stsp datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1877 876c234b 2018-09-10 stsp if (datalen != sizeof(ipack)) {
1878 876c234b 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
1879 876c234b 2018-09-10 stsp goto done;
1881 876c234b 2018-09-10 stsp memcpy(&ipack, imsg.data, sizeof(ipack));
1883 876c234b 2018-09-10 stsp pack->filesize = ipack.filesize;
1884 2c52c623 2024-01-30 op pack->fd = imsg_get_fd(&imsg);
1885 876c234b 2018-09-10 stsp if (pack->fd == -1) {
1886 2c52c623 2024-01-30 op err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1887 876c234b 2018-09-10 stsp goto done;
1889 56bef47a 2018-09-15 stsp if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1890 638f9024 2019-05-13 stsp err = got_error_from_errno("lseek");
1891 56bef47a 2018-09-15 stsp goto done;
1893 876c234b 2018-09-10 stsp pack->path_packfile = strdup(ipack.path_packfile);
1894 876c234b 2018-09-10 stsp if (pack->path_packfile == NULL) {
1895 638f9024 2019-05-13 stsp err = got_error_from_errno("strdup");
1896 ab2f42e7 2019-11-10 stsp goto done;
1899 dac5c75e 2022-06-04 stsp err = got_delta_cache_alloc(&pack->delta_cache);
1901 876c234b 2018-09-10 stsp goto done;
1903 876c234b 2018-09-10 stsp #ifndef GOT_PACK_NO_MMAP
1904 1c28a361 2022-10-25 op if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1905 1c28a361 2022-10-25 op pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1906 1c28a361 2022-10-25 op pack->fd, 0);
1907 1c28a361 2022-10-25 op if (pack->map == MAP_FAILED)
1908 1c28a361 2022-10-25 op pack->map = NULL; /* fall back to read(2) */
1912 876c234b 2018-09-10 stsp if (err) {
1913 2c52c623 2024-01-30 op if (pack != NULL)
1914 2c52c623 2024-01-30 op got_pack_close(pack);
1916 876c234b 2018-09-10 stsp *packp = pack;
1917 876c234b 2018-09-10 stsp imsg_free(&imsg);
1918 876c234b 2018-09-10 stsp return err;
1922 876c234b 2018-09-10 stsp main(int argc, char *argv[])
1924 876c234b 2018-09-10 stsp const struct got_error *err = NULL;
1925 876c234b 2018-09-10 stsp struct imsgbuf ibuf;
1926 876c234b 2018-09-10 stsp struct imsg imsg;
1927 c59b3346 2018-09-11 stsp struct got_packidx *packidx = NULL;
1928 c59b3346 2018-09-11 stsp struct got_pack *pack = NULL;
1929 c59b3346 2018-09-11 stsp struct got_object_cache objcache;
1930 67fd6849 2022-02-13 stsp FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1931 61af9b21 2022-06-28 stsp struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1932 d294b1dc 2022-10-18 stsp struct got_parsed_tree_entry *entries = NULL;
1933 d294b1dc 2022-10-18 stsp size_t nentries = 0, nentries_alloc = 0;
1935 876c234b 2018-09-10 stsp //static int attached;
1936 876c234b 2018-09-10 stsp //while (!attached) sleep(1);
1938 99437157 2018-11-11 stsp signal(SIGINT, catch_sigint);
1940 876c234b 2018-09-10 stsp imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1942 c59b3346 2018-09-11 stsp err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1943 c59b3346 2018-09-11 stsp if (err) {
1944 638f9024 2019-05-13 stsp err = got_error_from_errno("got_object_cache_init");
1945 c59b3346 2018-09-11 stsp got_privsep_send_error(&ibuf, err);
1949 2ff12563 2018-09-15 stsp #ifndef PROFILE
1950 876c234b 2018-09-10 stsp /* revoke access to most system calls */
1951 876c234b 2018-09-10 stsp if (pledge("stdio recvfd", NULL) == -1) {
1952 638f9024 2019-05-13 stsp err = got_error_from_errno("pledge");
1953 876c234b 2018-09-10 stsp got_privsep_send_error(&ibuf, err);
1958 876c234b 2018-09-10 stsp err = receive_packidx(&packidx, &ibuf);
1959 876c234b 2018-09-10 stsp if (err) {
1960 876c234b 2018-09-10 stsp got_privsep_send_error(&ibuf, err);
1964 876c234b 2018-09-10 stsp err = receive_pack(&pack, &ibuf);
1965 876c234b 2018-09-10 stsp if (err) {
1966 876c234b 2018-09-10 stsp got_privsep_send_error(&ibuf, err);
1971 99437157 2018-11-11 stsp if (sigint_received) {
1972 99437157 2018-11-11 stsp err = got_error(GOT_ERR_CANCELLED);
1976 876c234b 2018-09-10 stsp err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1977 876c234b 2018-09-10 stsp if (err) {
1978 876c234b 2018-09-10 stsp if (err->code == GOT_ERR_PRIVSEP_PIPE)
1979 876c234b 2018-09-10 stsp err = NULL;
1983 2c52c623 2024-01-30 op if (imsg.hdr.type == GOT_IMSG_STOP) {
1984 2c52c623 2024-01-30 op imsg_free(&imsg);
1988 876c234b 2018-09-10 stsp switch (imsg.hdr.type) {
1989 db696021 2022-01-04 stsp case GOT_IMSG_TMPFD:
1990 67fd6849 2022-02-13 stsp if (basefile == NULL) {
1991 67fd6849 2022-02-13 stsp err = receive_tempfile(&basefile, "w+",
1992 67fd6849 2022-02-13 stsp &imsg, &ibuf);
1993 67fd6849 2022-02-13 stsp } else if (accumfile == NULL) {
1994 67fd6849 2022-02-13 stsp err = receive_tempfile(&accumfile, "w+",
1995 67fd6849 2022-02-13 stsp &imsg, &ibuf);
1997 67fd6849 2022-02-13 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
1999 876c234b 2018-09-10 stsp case GOT_IMSG_PACKED_OBJECT_REQUEST:
2000 c59b3346 2018-09-11 stsp err = object_request(&imsg, &ibuf, pack, packidx,
2001 c59b3346 2018-09-11 stsp &objcache);
2003 59d1e4a0 2021-03-10 stsp case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
2004 db696021 2022-01-04 stsp if (basefile == NULL || accumfile == NULL) {
2005 db696021 2022-01-04 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
2008 59d1e4a0 2021-03-10 stsp err = raw_object_request(&imsg, &ibuf, pack, packidx,
2009 db696021 2022-01-04 stsp &objcache, basefile, accumfile);
2011 67fd6849 2022-02-13 stsp case GOT_IMSG_RAW_DELTA_OUTFD:
2012 67fd6849 2022-02-13 stsp if (delta_outfile != NULL) {
2013 67fd6849 2022-02-13 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
2016 67fd6849 2022-02-13 stsp err = receive_tempfile(&delta_outfile, "w",
2017 67fd6849 2022-02-13 stsp &imsg, &ibuf);
2019 67fd6849 2022-02-13 stsp case GOT_IMSG_RAW_DELTA_REQUEST:
2020 67fd6849 2022-02-13 stsp if (delta_outfile == NULL) {
2021 67fd6849 2022-02-13 stsp err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2024 67fd6849 2022-02-13 stsp err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2025 67fd6849 2022-02-13 stsp pack, packidx);
2027 fae7e038 2022-05-07 stsp case GOT_IMSG_DELTA_REUSE_REQUEST:
2028 24b7de1c 2022-12-03 stsp err = delta_reuse_request(&imsg, &ibuf, pack, packidx);
2030 876c234b 2018-09-10 stsp case GOT_IMSG_COMMIT_REQUEST:
2031 c59b3346 2018-09-11 stsp err = commit_request(&imsg, &ibuf, pack, packidx,
2032 7762fe12 2018-11-05 stsp &objcache);
2034 876c234b 2018-09-10 stsp case GOT_IMSG_TREE_REQUEST:
2035 c59b3346 2018-09-11 stsp err = tree_request(&imsg, &ibuf, pack, packidx,
2036 d294b1dc 2022-10-18 stsp &objcache, &entries, &nentries, &nentries_alloc);
2038 876c234b 2018-09-10 stsp case GOT_IMSG_BLOB_REQUEST:
2039 db696021 2022-01-04 stsp if (basefile == NULL || accumfile == NULL) {
2040 db696021 2022-01-04 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
2043 c59b3346 2018-09-11 stsp err = blob_request(&imsg, &ibuf, pack, packidx,
2044 db696021 2022-01-04 stsp &objcache, basefile, accumfile);
2046 f4a881ce 2018-11-17 stsp case GOT_IMSG_TAG_REQUEST:
2047 f4a881ce 2018-11-17 stsp err = tag_request(&imsg, &ibuf, pack, packidx,
2048 62d463ca 2020-10-20 naddy &objcache);
2050 ca6e02ac 2020-01-07 stsp case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2051 ca6e02ac 2020-01-07 stsp err = commit_traversal_request(&imsg, &ibuf, pack,
2052 ca6e02ac 2020-01-07 stsp packidx, &objcache);
2054 0ab4c957 2022-06-13 stsp case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2055 0ab4c957 2022-06-13 stsp err = enumeration_request(&imsg, &ibuf, pack,
2056 0ab4c957 2022-06-13 stsp packidx, &objcache);
2058 61af9b21 2022-06-28 stsp case GOT_IMSG_COMMIT_PAINTING_INIT:
2059 61af9b21 2022-06-28 stsp commit_painting_free(&keep, &drop, &skip);
2060 61af9b21 2022-06-28 stsp err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2062 61af9b21 2022-06-28 stsp case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2063 61af9b21 2022-06-28 stsp if (keep == NULL || drop == NULL || skip == NULL) {
2064 61af9b21 2022-06-28 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
2067 61af9b21 2022-06-28 stsp err = commit_painting_request(&imsg, &ibuf, pack,
2068 61af9b21 2022-06-28 stsp packidx, &objcache, keep, drop, skip);
2070 61af9b21 2022-06-28 stsp case GOT_IMSG_COMMIT_PAINTING_DONE:
2071 61af9b21 2022-06-28 stsp commit_painting_free(&keep, &drop, &skip);
2074 876c234b 2018-09-10 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
2078 876c234b 2018-09-10 stsp imsg_free(&imsg);
2083 d294b1dc 2022-10-18 stsp free(entries);
2084 61af9b21 2022-06-28 stsp commit_painting_free(&keep, &drop, &skip);
2085 c59b3346 2018-09-11 stsp if (packidx)
2086 c59b3346 2018-09-11 stsp got_packidx_close(packidx);
2088 c59b3346 2018-09-11 stsp got_pack_close(pack);
2089 48d5fe42 2018-09-15 stsp got_object_cache_close(&objcache);
2090 876c234b 2018-09-10 stsp imsg_clear(&ibuf);
2091 db696021 2022-01-04 stsp if (basefile && fclose(basefile) == EOF && err == NULL)
2092 db696021 2022-01-04 stsp err = got_error_from_errno("fclose");
2093 db696021 2022-01-04 stsp if (accumfile && fclose(accumfile) == EOF && err == NULL)
2094 db696021 2022-01-04 stsp err = got_error_from_errno("fclose");
2095 67fd6849 2022-02-13 stsp if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2096 67fd6849 2022-02-13 stsp err = got_error_from_errno("fclose");
2097 99437157 2018-11-11 stsp if (err) {
2098 80d5f134 2018-11-11 stsp if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2099 80d5f134 2018-11-11 stsp fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2100 99437157 2018-11-11 stsp got_privsep_send_error(&ibuf, err);
2103 08578a35 2021-01-22 stsp if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2104 638f9024 2019-05-13 stsp err = got_error_from_errno("close");
2105 876c234b 2018-09-10 stsp return err ? 1 : 0;
