Got Mirror

Blob

Date:: Thu Nov 14 17:12:32 2019 UTC
Message:: reduce the amount of memcpy() and strdup() while parsing tree entries
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/syslimits.h>
22 #include <sys/mman.h>
23 
24 #include <limits.h>
25 #include <signal.h>
26 #include <stdint.h>
27 #include <imsg.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <sha1.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
37 
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_privsep.h"
44 #include "got_lib_pack.h"
45 
46 static volatile sig_atomic_t sigint_received;
47 
48 static void
49 catch_sigint(int signo)
50 {
51 	sigint_received = 1;
52 }
53 
54 static const struct got_error *
55 open_object(struct got_object **obj, struct got_pack *pack,
56     struct got_packidx *packidx, int idx, struct got_object_id *id,
57     struct got_object_cache *objcache)
58 {
59 	const struct got_error *err;
60 
61 	err = got_packfile_open_object(obj, pack, packidx, idx, id);
62 	if (err)
63 		return err;
64 	(*obj)->refcnt++;
65 
66 	err = got_object_cache_add(objcache, id, *obj);
67 	if (err) {
68 		if (err->code == GOT_ERR_OBJ_EXISTS ||
69 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
70 			err = NULL;
71 		return err;
72 	}
73 	(*obj)->refcnt++;
74 	return NULL;
75 }
76 
77 static const struct got_error *
78 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
79     struct got_packidx *packidx, struct got_object_cache *objcache)
80 {
81 	const struct got_error *err = NULL;
82 	struct got_imsg_packed_object iobj;
83 	struct got_object *obj;
84 	struct got_object_id id;
85 	size_t datalen;
86 
87 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
88 	if (datalen != sizeof(iobj))
89 		return got_error(GOT_ERR_PRIVSEP_LEN);
90 	memcpy(&iobj, imsg->data, sizeof(iobj));
91 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
92 
93 	obj = got_object_cache_get(objcache, &id);
94 	if (obj) {
95 		obj->refcnt++;
96 	} else {
97 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
98 		    objcache);
99 		if (err)
100 			goto done;
101 	}
102 
103 	err = got_privsep_send_obj(ibuf, obj);
104 done:
105 	got_object_close(obj);
106 	return err;
107 }
108 
109 static const struct got_error *
110 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
111     struct got_packidx *packidx, struct got_object_cache *objcache)
112 {
113 	const struct got_error *err = NULL;
114 	struct got_imsg_packed_object iobj;
115 	struct got_object *obj = NULL;
116 	struct got_commit_object *commit = NULL;
117 	uint8_t *buf = NULL;
118 	size_t len;
119 	struct got_object_id id;
120 	size_t datalen;
121 
122 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
123 	if (datalen != sizeof(iobj))
124 		return got_error(GOT_ERR_PRIVSEP_LEN);
125 	memcpy(&iobj, imsg->data, sizeof(iobj));
126 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
127 
128 	obj = got_object_cache_get(objcache, &id);
129 	if (obj) {
130 		obj->refcnt++;
131 	} else {
132 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
133 		    objcache);
134 		if (err)
135 			return err;
136 	}
137 
138 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
139 	if (err)
140 		goto done;
141 
142 	obj->size = len;
143 	err = got_object_parse_commit(&commit, buf, len);
144 	if (err)
145 		goto done;
146 
147 	err = got_privsep_send_commit(ibuf, commit);
148 done:
149 	free(buf);
150 	got_object_close(obj);
151 	if (commit)
152 		got_object_commit_close(commit);
153 	if (err) {
154 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
155 			err = NULL;
156 		else
157 			got_privsep_send_error(ibuf, err);
158 	}
159 
160 	return err;
161 }
162 
163 static const struct got_error *
164 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
165     struct got_packidx *packidx, struct got_object_cache *objcache)
166 {
167 	const struct got_error *err = NULL;
168 	struct got_imsg_packed_object iobj;
169 	struct got_object *obj = NULL;
170 	struct got_pathlist_head entries;
171 	int nentries = 0;
172 	uint8_t *buf = NULL;
173 	size_t len;
174 	struct got_object_id id;
175 	size_t datalen;
176 
177 	TAILQ_INIT(&entries);
178 
179 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
180 	if (datalen != sizeof(iobj))
181 		return got_error(GOT_ERR_PRIVSEP_LEN);
182 	memcpy(&iobj, imsg->data, sizeof(iobj));
183 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
184 
185 	obj = got_object_cache_get(objcache, &id);
186 	if (obj) {
187 		obj->refcnt++;
188 	} else {
189 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
190 		    objcache);
191 		if (err)
192 			return err;
193 	}
194 
195 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
196 	if (err)
197 		goto done;
198 
199 	obj->size = len;
200 	err = got_object_parse_tree(&entries, &nentries, buf, len);
201 	if (err)
202 		goto done;
203 
204 	err = got_privsep_send_tree(ibuf, &entries, nentries);
205 done:
206 	got_pathlist_free(&entries);
207 	free(buf);
208 	got_object_close(obj);
209 	if (err) {
210 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
211 			err = NULL;
212 		else
213 			got_privsep_send_error(ibuf, err);
214 	}
215 
216 	return err;
217 }
218 
219 static const struct got_error *
220 receive_file(FILE **f, struct imsgbuf *ibuf, int imsg_code)
221 {
222 	const struct got_error *err;
223 	struct imsg imsg;
224 	size_t datalen;
225 
226 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
227 	if (err)
228 		return err;
229 
230 	if (imsg.hdr.type != imsg_code) {
231 		err = got_error(GOT_ERR_PRIVSEP_MSG);
232 		goto done;
233 	}
234 
235 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
236 	if (datalen != 0) {
237 		err = got_error(GOT_ERR_PRIVSEP_LEN);
238 		goto done;
239 	}
240 	if (imsg.fd == -1) {
241 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
242 		goto done;
243 	}
244 
245 	*f = fdopen(imsg.fd, "w+");
246 	if (*f == NULL) {
247 		err = got_error_from_errno("fdopen");
248 		close(imsg.fd);
249 		goto done;
250 	}
251 done:
252 	imsg_free(&imsg);
253 	return err;
254 }
255 
256 static const struct got_error *
257 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
258     struct got_packidx *packidx, struct got_object_cache *objcache)
259 {
260 	const struct got_error *err = NULL;
261 	struct got_imsg_packed_object iobj;
262 	struct got_object *obj = NULL;
263 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
264 	struct got_object_id id;
265 	size_t datalen;
266 	uint64_t blob_size;
267 	uint8_t *buf = NULL;
268 
269 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
270 	if (datalen != sizeof(iobj))
271 		return got_error(GOT_ERR_PRIVSEP_LEN);
272 	memcpy(&iobj, imsg->data, sizeof(iobj));
273 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
274 
275 	obj = got_object_cache_get(objcache, &id);
276 	if (obj) {
277 		obj->refcnt++;
278 	} else {
279 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
280 		    objcache);
281 		if (err)
282 			return err;
283 	}
284 
285 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
286 	if (err)
287 		goto done;
288 	err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD);
289 	if (err)
290 		goto done;
291 	err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD);
292 	if (err)
293 		goto done;
294 
295 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
296 		err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
297 		if (err)
298 			goto done;
299 	} else
300 		blob_size = obj->size;
301 
302 	if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
303 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
304 		    obj, pack);
305 	else
306 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
307 		    accumfile);
308 	if (err)
309 		goto done;
310 
311 	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
312 done:
313 	free(buf);
314 	if (outfile && fclose(outfile) != 0 && err == NULL)
315 		err = got_error_from_errno("fclose");
316 	if (basefile && fclose(basefile) != 0 && err == NULL)
317 		err = got_error_from_errno("fclose");
318 	if (accumfile && fclose(accumfile) != 0 && err == NULL)
319 		err = got_error_from_errno("fclose");
320 	got_object_close(obj);
321 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
322 		got_privsep_send_error(ibuf, err);
323 
324 	return err;
325 }
326 
327 static const struct got_error *
328 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
329     struct got_packidx *packidx, struct got_object_cache *objcache)
330 {
331 	const struct got_error *err = NULL;
332 	struct got_imsg_packed_object iobj;
333 	struct got_object *obj = NULL;
334 	struct got_tag_object *tag = NULL;
335 	uint8_t *buf = NULL;
336 	size_t len;
337 	struct got_object_id id;
338 	size_t datalen;
339 
340 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
341 	if (datalen != sizeof(iobj))
342 		return got_error(GOT_ERR_PRIVSEP_LEN);
343 	memcpy(&iobj, imsg->data, sizeof(iobj));
344 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
345 
346 	obj = got_object_cache_get(objcache, &id);
347 	if (obj) {
348 		obj->refcnt++;
349 	} else {
350 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
351 		    objcache);
352 		if (err)
353 			return err;
354 	}
355 
356 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
357 	if (err)
358 		goto done;
359 
360 	obj->size = len;
361 	err = got_object_parse_tag(&tag, buf, len);
362 	if (err)
363 		goto done;
364 
365 	err = got_privsep_send_tag(ibuf, tag);
366 done:
367 	free(buf);
368 	got_object_close(obj);
369 	if (tag)
370 		got_object_tag_close(tag);
371 	if (err) {
372 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
373 			err = NULL;
374 		else
375 			got_privsep_send_error(ibuf, err);
376 	}
377 
378 	return err;
379 }
380 
381 static const struct got_error *
382 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
383 {
384 	const struct got_error *err = NULL;
385 	struct imsg imsg;
386 	struct got_imsg_packidx ipackidx;
387 	size_t datalen;
388 	struct got_packidx *p;
389 
390 	*packidx = NULL;
391 
392 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
393 	if (err)
394 		return err;
395 
396 	p = calloc(1, sizeof(*p));
397 	if (p == NULL) {
398 		err = got_error_from_errno("calloc");
399 		goto done;
400 	}
401 
402 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
403 		err = got_error(GOT_ERR_PRIVSEP_MSG);
404 		goto done;
405 	}
406 
407 	if (imsg.fd == -1) {
408 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
409 		goto done;
410 	}
411 
412 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
413 	if (datalen != sizeof(ipackidx)) {
414 		err = got_error(GOT_ERR_PRIVSEP_LEN);
415 		goto done;
416 	}
417 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
418 
419 	p->len = ipackidx.len;
420 	p->fd = dup(imsg.fd);
421 	if (p->fd == -1) {
422 		err = got_error_from_errno("dup");
423 		goto done;
424 	}
425 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
426 		err = got_error_from_errno("lseek");
427 		goto done;
428 	}
429 
430 #ifndef GOT_PACK_NO_MMAP
431 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
432 	if (p->map == MAP_FAILED)
433 		p->map = NULL; /* fall back to read(2) */
434 #endif
435 	err = got_packidx_init_hdr(p, 1);
436 done:
437 	if (err) {
438 		if (imsg.fd != -1)
439 			close(imsg.fd);
440 		got_packidx_close(p);
441 	} else
442 		*packidx = p;
443 	imsg_free(&imsg);
444 	return err;
445 }
446 
447 static const struct got_error *
448 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
449 {
450 	const struct got_error *err = NULL;
451 	struct imsg imsg;
452 	struct got_imsg_pack ipack;
453 	size_t datalen;
454 	struct got_pack *pack;
455 
456 	*packp = NULL;
457 
458 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
459 	if (err)
460 		return err;
461 
462 	pack = calloc(1, sizeof(*pack));
463 	if (pack == NULL) {
464 		err = got_error_from_errno("calloc");
465 		goto done;
466 	}
467 
468 	if (imsg.hdr.type != GOT_IMSG_PACK) {
469 		err = got_error(GOT_ERR_PRIVSEP_MSG);
470 		goto done;
471 	}
472 
473 	if (imsg.fd == -1) {
474 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
475 		goto done;
476 	}
477 
478 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
479 	if (datalen != sizeof(ipack)) {
480 		err = got_error(GOT_ERR_PRIVSEP_LEN);
481 		goto done;
482 	}
483 	memcpy(&ipack, imsg.data, sizeof(ipack));
484 
485 	pack->filesize = ipack.filesize;
486 	pack->fd = dup(imsg.fd);
487 	if (pack->fd == -1) {
488 		err = got_error_from_errno("dup");
489 		goto done;
490 	}
491 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
492 		err = got_error_from_errno("lseek");
493 		goto done;
494 	}
495 	pack->path_packfile = strdup(ipack.path_packfile);
496 	if (pack->path_packfile == NULL) {
497 		err = got_error_from_errno("strdup");
498 		goto done;
499 	}
500 
501 	pack->delta_cache = got_delta_cache_alloc(100,
502 	    GOT_DELTA_RESULT_SIZE_CACHED_MAX);
503 	if (pack->delta_cache == NULL) {
504 		err = got_error_from_errno("got_delta_cache_alloc");
505 		goto done;
506 	}
507 
508 #ifndef GOT_PACK_NO_MMAP
509 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
510 	    pack->fd, 0);
511 	if (pack->map == MAP_FAILED)
512 		pack->map = NULL; /* fall back to read(2) */
513 #endif
514 done:
515 	if (err) {
516 		if (imsg.fd != -1)
517 			close(imsg.fd);
518 		free(pack);
519 	} else
520 		*packp = pack;
521 	imsg_free(&imsg);
522 	return err;
523 }
524 
525 int
526 main(int argc, char *argv[])
527 {
528 	const struct got_error *err = NULL;
529 	struct imsgbuf ibuf;
530 	struct imsg imsg;
531 	struct got_packidx *packidx = NULL;
532 	struct got_pack *pack = NULL;
533 	struct got_object_cache objcache;
534 
535 	//static int attached;
536 	//while (!attached) sleep(1);
537 
538 	signal(SIGINT, catch_sigint);
539 
540 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
541 
542 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
543 	if (err) {
544 		err = got_error_from_errno("got_object_cache_init");
545 		got_privsep_send_error(&ibuf, err);
546 		return 1;
547 	}
548 
549 #ifndef PROFILE
550 	/* revoke access to most system calls */
551 	if (pledge("stdio recvfd", NULL) == -1) {
552 		err = got_error_from_errno("pledge");
553 		got_privsep_send_error(&ibuf, err);
554 		return 1;
555 	}
556 #endif
557 
558 	err = receive_packidx(&packidx, &ibuf);
559 	if (err) {
560 		got_privsep_send_error(&ibuf, err);
561 		return 1;
562 	}
563 
564 	err = receive_pack(&pack, &ibuf);
565 	if (err) {
566 		got_privsep_send_error(&ibuf, err);
567 		return 1;
568 	}
569 
570 	for (;;) {
571 		imsg.fd = -1;
572 
573 		if (sigint_received) {
574 			err = got_error(GOT_ERR_CANCELLED);
575 			break;
576 		}
577 
578 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
579 		if (err) {
580 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
581 				err = NULL;
582 			break;
583 		}
584 
585 		if (imsg.hdr.type == GOT_IMSG_STOP)
586 			break;
587 
588 		switch (imsg.hdr.type) {
589 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
590 			err = object_request(&imsg, &ibuf, pack, packidx,
591 			    &objcache);
592 			break;
593 		case GOT_IMSG_COMMIT_REQUEST:
594 			err = commit_request(&imsg, &ibuf, pack, packidx,
595 			    &objcache);
596 			break;
597 		case GOT_IMSG_TREE_REQUEST:
598 			err = tree_request(&imsg, &ibuf, pack, packidx,
599 			   &objcache);
600 			break;
601 		case GOT_IMSG_BLOB_REQUEST:
602 			err = blob_request(&imsg, &ibuf, pack, packidx,
603 			   &objcache);
604 			break;
605 		case GOT_IMSG_TAG_REQUEST:
606 			err = tag_request(&imsg, &ibuf, pack, packidx,
607 			   &objcache);
608 			break;
609 		default:
610 			err = got_error(GOT_ERR_PRIVSEP_MSG);
611 			break;
612 		}
613 
614 		if (imsg.fd != -1 && close(imsg.fd) != 0 && err == NULL)
615 			err = got_error_from_errno("close");
616 		imsg_free(&imsg);
617 		if (err)
618 			break;
619 	}
620 
621 	if (packidx)
622 		got_packidx_close(packidx);
623 	if (pack)
624 		got_pack_close(pack);
625 	got_object_cache_close(&objcache);
626 	imsg_clear(&ibuf);
627 	if (err) {
628 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
629 			fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
630 			got_privsep_send_error(&ibuf, err);
631 		}
632 	}
633 	if (close(GOT_IMSG_FD_CHILD) != 0 && err == NULL)
634 		err = got_error_from_errno("close");
635 	return err ? 1 : 0;
636 }