Got Mirror

Blob

Date:: Mon Jan 9 17:32:29 2023 UTC
Message:: use a caller-specified size limit for mapped files in got_object_raw_alloc() Without this we end up being confused about whether a raw object has been mapped into memory, leading to crashes. ok op@
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2022 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <errno.h>
22 #include <limits.h>
23 #include <sha1.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <unistd.h>
28 
29 #include "got_error.h"
30 #include "got_object.h"
31 #include "got_repository.h"
32 #include "got_path.h"
33 
34 #include "got_lib_delta.h"
35 #include "got_lib_object.h"
36 #include "got_lib_object_cache.h"
37 #include "got_lib_object_parse.h"
38 #include "got_lib_pack.h"
39 #include "got_lib_repository.h"
40 
41 const struct got_error *
42 got_object_open_packed(struct got_object **obj, struct got_object_id *id,
43     struct got_repository *repo)
44 {
45 	const struct got_error *err = NULL;
46 	struct got_pack *pack = NULL;
47 	struct got_packidx *packidx = NULL;
48 	int idx;
49 	char *path_packfile;
50 
51 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
52 	if (err)
53 		return err;
54 
55 	err = got_packidx_get_packfile_path(&path_packfile,
56 	    packidx->path_packidx);
57 	if (err)
58 		return err;
59 
60 	pack = got_repo_get_cached_pack(repo, path_packfile);
61 	if (pack == NULL) {
62 		err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
63 		if (err)
64 			goto done;
65 	}
66 
67 	err = got_packfile_open_object(obj, pack, packidx, idx, id);
68 	if (err)
69 		return err;
70 	(*obj)->refcnt++;
71 
72 	err = got_repo_cache_object(repo, id, *obj);
73 	if (err) {
74 		if (err->code == GOT_ERR_OBJ_EXISTS ||
75 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
76 			err = NULL;
77 	}
78 done:
79 	free(path_packfile);
80 	return err;
81 }
82 
83 const struct got_error *
84 got_object_open_from_packfile(struct got_object **obj, struct got_object_id *id,
85     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
86     struct got_repository *repo)
87 {
88 	return got_error(GOT_ERR_NOT_IMPL);
89 }
90 
91 const struct got_error *
92 got_object_read_raw_delta(uint64_t *base_size, uint64_t *result_size,
93     off_t *delta_size, off_t *delta_compressed_size, off_t *delta_offset,
94     off_t *delta_out_offset, struct got_object_id **base_id, int delta_cache_fd,
95     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
96     struct got_repository *repo)
97 {
98 	return got_error(GOT_ERR_NOT_IMPL);
99 }
100 
101 const struct got_error *
102 got_object_open(struct got_object **obj, struct got_repository *repo,
103     struct got_object_id *id)
104 {
105 	const struct got_error *err = NULL;
106 	int fd;
107 
108 	*obj = got_repo_get_cached_object(repo, id);
109 	if (*obj != NULL) {
110 		(*obj)->refcnt++;
111 		return NULL;
112 	}
113 
114 	err = got_object_open_packed(obj, id, repo);
115 	if (err) {
116 		if (err->code != GOT_ERR_NO_OBJ)
117 			return err;
118 	} else
119 		return NULL;
120 
121 	err = got_object_open_loose_fd(&fd, id, repo);
122 	if (err) {
123 		if (err->code == GOT_ERR_ERRNO && errno == ENOENT)
124 			err = got_error_no_obj(id);
125 		return err;
126 	}
127 
128 	err = got_object_read_header(obj, fd);
129 	if (err)
130 		goto done;
131 
132 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
133 	(*obj)->refcnt++;
134 
135 	err = got_repo_cache_object(repo, id, *obj);
136 	if (err) {
137 		if (err->code == GOT_ERR_OBJ_EXISTS ||
138 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
139 			err = NULL;
140 	}
141 done:
142 	if (close(fd) == -1 && err == NULL)
143 		err = got_error_from_errno("close");
144 	return err;
145 }
146 
147 static const struct got_error *
148 wrap_fd(FILE **f, int wrapped_fd)
149 {
150 	const struct got_error *err = NULL;
151 	int fd;
152 
153 	if (ftruncate(wrapped_fd, 0L) == -1)
154 		return got_error_from_errno("ftruncate");
155 
156 	if (lseek(wrapped_fd, 0L, SEEK_SET) == -1)
157 		return got_error_from_errno("lseek");
158 
159 	fd = dup(wrapped_fd);
160 	if (fd == -1)
161 		return got_error_from_errno("dup");
162 
163 	*f = fdopen(fd, "w+");
164 	if (*f == NULL) {
165 		err = got_error_from_errno("fdopen");
166 		close(fd);
167 	}
168 	return err;
169 }
170 
171 static const struct got_error *
172 read_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen,
173     int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
174     struct got_object_id *id)
175 {
176 	const struct got_error *err = NULL;
177 	uint64_t raw_size = 0;
178 	struct got_object *obj;
179 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
180 
181 	*outbuf = NULL;
182 	*size = 0;
183 	*hdrlen = 0;
184 
185 	err = got_packfile_open_object(&obj, pack, packidx, idx, id);
186 	if (err)
187 		return err;
188 
189 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
190 		err = got_pack_get_max_delta_object_size(&raw_size, obj, pack);
191 		if (err)
192 			goto done;
193 	} else
194 		raw_size = obj->size;
195 
196 	if (raw_size <= GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
197 		size_t len;
198 		err = got_packfile_extract_object_to_mem(outbuf, &len,
199 		    obj, pack);
200 		if (err)
201 			goto done;
202 		*size = (off_t)len;
203 	} else {
204 		/*
205 		 * XXX This uses 3 file extra descriptors for no good reason.
206 		 * We should have got_packfile_extract_object_to_fd().
207 		 */
208 		err = wrap_fd(&outfile, outfd);
209 		if (err)
210 			goto done;
211 		err = wrap_fd(&basefile, pack->basefd);
212 		if (err)
213 			goto done;
214 		err = wrap_fd(&accumfile, pack->accumfd);
215 		if (err)
216 			goto done;
217 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
218 		    accumfile);
219 		if (err)
220 			goto done;
221 		*size = obj->size;
222 	}
223 
224 	*hdrlen = obj->hdrlen;
225 done:
226 	got_object_close(obj);
227 	if (outfile && fclose(outfile) == EOF && err == NULL)
228 		err = got_error_from_errno("fclose");
229 	if (basefile && fclose(basefile) == EOF && err == NULL)
230 		err = got_error_from_errno("fclose");
231 	if (accumfile && fclose(accumfile) == EOF && err == NULL)
232 		err = got_error_from_errno("fclose");
233 	return err;
234 
235 }
236 
237 static void
238 put_raw_object_tempfile(struct got_raw_object *obj)
239 {
240 	struct got_repository *repo = obj->close_arg;
241 
242 	if (obj->tempfile_idx != -1)
243 		got_repo_temp_fds_put(obj->tempfile_idx, repo);
244 }
245 
246 /* *outfd must be initialized to -1 by caller */
247 const struct got_error *
248 got_object_raw_open(struct got_raw_object **obj, int *outfd,
249     struct got_repository *repo, struct got_object_id *id)
250 {
251 	const struct got_error *err = NULL;
252 	struct got_packidx *packidx = NULL;
253 	int idx, tempfile_idx = -1;
254 	uint8_t *outbuf = NULL;
255 	off_t size = 0;
256 	size_t hdrlen = 0;
257 	char *path_packfile = NULL;
258 
259 	*obj = got_repo_get_cached_raw_object(repo, id);
260 	if (*obj != NULL) {
261 		(*obj)->refcnt++;
262 		return NULL;
263 	}
264 
265 	if (*outfd == -1) {
266 		int tempfd;
267 
268 		err = got_repo_temp_fds_get(&tempfd, &tempfile_idx, repo);
269 		if (err)
270 			return err;
271 
272 		/* Duplicate tempfile descriptor to allow use of fdopen(3). */
273 		*outfd = dup(tempfd);
274 		if (*outfd == -1) {
275 			got_repo_temp_fds_put(tempfile_idx, repo);
276 			return got_error_from_errno("dup");
277 		}
278 	}
279 
280 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
281 	if (err == NULL) {
282 		struct got_pack *pack = NULL;
283 
284 		err = got_packidx_get_packfile_path(&path_packfile,
285 		    packidx->path_packidx);
286 		if (err)
287 			goto done;
288 
289 		pack = got_repo_get_cached_pack(repo, path_packfile);
290 		if (pack == NULL) {
291 			err = got_repo_cache_pack(&pack, repo, path_packfile,
292 			    packidx);
293 			if (err)
294 				goto done;
295 		}
296 		err = read_packed_object_raw(&outbuf, &size, &hdrlen,
297 		    *outfd, pack, packidx, idx, id);
298 		if (err)
299 			goto done;
300 	} else if (err->code == GOT_ERR_NO_OBJ) {
301 		int fd;
302 
303 		err = got_object_open_loose_fd(&fd, id, repo);
304 		if (err)
305 			goto done;
306 		err = got_object_read_raw(&outbuf, &size, &hdrlen,
307 		    GOT_DELTA_RESULT_SIZE_CACHED_MAX, *outfd, id, fd);
308 		if (close(fd) == -1 && err == NULL)
309 			err = got_error_from_errno("close");
310 		if (err)
311 			goto done;
312 	}
313 
314 	err = got_object_raw_alloc(obj, outbuf, outfd,
315 	    GOT_DELTA_RESULT_SIZE_CACHED_MAX, hdrlen, size);
316 	if (err)
317 		goto done;
318 
319 	err = got_repo_cache_raw_object(repo, id, *obj);
320 	if (err) {
321 		if (err->code == GOT_ERR_OBJ_EXISTS ||
322 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
323 			err = NULL;
324 	}
325 done:
326 	free(path_packfile);
327 	if (err) {
328 		if (*obj) {
329 			got_object_raw_close(*obj);
330 			*obj = NULL;
331 		}
332 		free(outbuf);
333 		if (tempfile_idx != -1)
334 			got_repo_temp_fds_put(tempfile_idx, repo);
335 	} else {
336 		(*obj)->tempfile_idx = tempfile_idx;
337 		(*obj)->close_cb = put_raw_object_tempfile;
338 		(*obj)->close_arg = repo;
339 	}
340 	return err;
341 }
342 
343 static const struct got_error *
344 open_commit(struct got_commit_object **commit,
345     struct got_repository *repo, struct got_object_id *id, int check_cache)
346 {
347 	const struct got_error *err = NULL;
348 	struct got_packidx *packidx = NULL;
349 	int idx;
350 	char *path_packfile = NULL;
351 
352 	if (check_cache) {
353 		*commit = got_repo_get_cached_commit(repo, id);
354 		if (*commit != NULL) {
355 			(*commit)->refcnt++;
356 			return NULL;
357 		}
358 	} else
359 		*commit = NULL;
360 
361 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
362 	if (err == NULL) {
363 		struct got_pack *pack = NULL;
364 		struct got_object *obj;
365 		uint8_t *buf;
366 		size_t len;
367 
368 		err = got_packidx_get_packfile_path(&path_packfile,
369 		    packidx->path_packidx);
370 		if (err)
371 			return err;
372 
373 		pack = got_repo_get_cached_pack(repo, path_packfile);
374 		if (pack == NULL) {
375 			err = got_repo_cache_pack(&pack, repo, path_packfile,
376 			    packidx);
377 			if (err)
378 				goto done;
379 		}
380 		err = got_packfile_open_object(&obj, pack, packidx, idx, id);
381 		if (err)
382 			goto done;
383 		err = got_packfile_extract_object_to_mem(&buf, &len,
384 		    obj, pack);
385 		got_object_close(obj);
386 		if (err)
387 			goto done;
388 		err = got_object_parse_commit(commit, buf, len);
389 		free(buf);
390 	} else if (err->code == GOT_ERR_NO_OBJ) {
391 		int fd;
392 
393 		err = got_object_open_loose_fd(&fd, id, repo);
394 		if (err)
395 			return err;
396 		err = got_object_read_commit(commit, fd, id, 0);
397 		if (close(fd) == -1 && err == NULL)
398 			err = got_error_from_errno("close");
399 		if (err)
400 			return err;
401 	}
402 
403 	if (err == NULL) {
404 		(*commit)->refcnt++;
405 		err = got_repo_cache_commit(repo, id, *commit);
406 		if (err) {
407 			if (err->code == GOT_ERR_OBJ_EXISTS ||
408 			    err->code == GOT_ERR_OBJ_TOO_LARGE)
409 				err = NULL;
410 		}
411 	}
412 done:
413 	free(path_packfile);
414 	return err;
415 }
416 
417 const struct got_error *
418 got_object_open_as_commit(struct got_commit_object **commit,
419     struct got_repository *repo, struct got_object_id *id)
420 {
421 	*commit = got_repo_get_cached_commit(repo, id);
422 	if (*commit != NULL) {
423 		(*commit)->refcnt++;
424 		return NULL;
425 	}
426 
427 	return open_commit(commit, repo, id, 0);
428 }
429 
430 const struct got_error *
431 got_object_commit_open(struct got_commit_object **commit,
432     struct got_repository *repo, struct got_object *obj)
433 {
434 	return open_commit(commit, repo, got_object_get_id(obj), 1);
435 }
436 
437 static const struct got_error *
438 open_tree(struct got_tree_object **tree,
439     struct got_repository *repo, struct got_object_id *id, int check_cache)
440 {
441 	const struct got_error *err = NULL;
442 	struct got_packidx *packidx = NULL;
443 	int idx;
444 	char *path_packfile = NULL;
445 	struct got_parsed_tree_entry *entries = NULL;
446 	size_t nentries = 0, nentries_alloc = 0, i;
447 	uint8_t *buf = NULL;
448 
449 	if (check_cache) {
450 		*tree = got_repo_get_cached_tree(repo, id);
451 		if (*tree != NULL) {
452 			(*tree)->refcnt++;
453 			return NULL;
454 		}
455 	} else
456 		*tree = NULL;
457 
458 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
459 	if (err == NULL) {
460 		struct got_pack *pack = NULL;
461 		struct got_object *obj;
462 		size_t len;
463 
464 		err = got_packidx_get_packfile_path(&path_packfile,
465 		    packidx->path_packidx);
466 		if (err)
467 			return err;
468 
469 		pack = got_repo_get_cached_pack(repo, path_packfile);
470 		if (pack == NULL) {
471 			err = got_repo_cache_pack(&pack, repo, path_packfile,
472 			    packidx);
473 			if (err)
474 				goto done;
475 		}
476 		err = got_packfile_open_object(&obj, pack, packidx, idx, id);
477 		if (err)
478 			goto done;
479 		err = got_packfile_extract_object_to_mem(&buf, &len,
480 		    obj, pack);
481 		got_object_close(obj);
482 		if (err)
483 			goto done;
484 		err = got_object_parse_tree(&entries, &nentries,
485 		    &nentries_alloc, buf, len);
486 		if (err)
487 			goto done;
488 	} else if (err->code == GOT_ERR_NO_OBJ) {
489 		int fd;
490 
491 		err = got_object_open_loose_fd(&fd, id, repo);
492 		if (err)
493 			return err;
494 		err = got_object_read_tree(&entries, &nentries,
495 		    &nentries_alloc, &buf, fd, id);
496 		if (close(fd) == -1 && err == NULL)
497 			err = got_error_from_errno("close");
498 		if (err)
499 			goto done;
500 	} else
501 		goto done;
502 
503 	*tree = malloc(sizeof(**tree));
504 	if (*tree == NULL) {
505 		err = got_error_from_errno("malloc");
506 		goto done;
507 	}
508 	(*tree)->entries = calloc(nentries, sizeof(struct got_tree_entry));
509 	if ((*tree)->entries == NULL) {
510 		err = got_error_from_errno("malloc");
511 		goto done;
512 	}
513 	(*tree)->nentries = nentries;
514 	(*tree)->refcnt = 0;
515 
516 	for (i = 0; i < nentries; i++) {
517 		struct got_parsed_tree_entry *pe = &entries[i];
518 		struct got_tree_entry *te = &(*tree)->entries[i];
519 	
520 		if (strlcpy(te->name, pe->name,
521 		    sizeof(te->name)) >= sizeof(te->name)) {
522 			err = got_error(GOT_ERR_NO_SPACE);
523 			goto done;
524 		}
525 		memcpy(te->id.sha1, pe->id, SHA1_DIGEST_LENGTH);
526 		te->mode = pe->mode;
527 		te->idx = i;
528 	}
529 done:
530 	free(path_packfile);
531 	free(entries);
532 	free(buf);
533 	if (err == NULL) {
534 		(*tree)->refcnt++;
535 		err = got_repo_cache_tree(repo, id, *tree);
536 		if (err) {
537 			if (err->code == GOT_ERR_OBJ_EXISTS ||
538 			    err->code == GOT_ERR_OBJ_TOO_LARGE)
539 				err = NULL;
540 		}
541 	}
542 	if (err) {
543 		if (*tree)
544 			free((*tree)->entries);
545 		free(*tree);
546 		*tree = NULL;
547 	}
548 	return err;
549 }
550 
551 const struct got_error *
552 got_object_open_as_tree(struct got_tree_object **tree,
553     struct got_repository *repo, struct got_object_id *id)
554 {
555 	*tree = got_repo_get_cached_tree(repo, id);
556 	if (*tree != NULL) {
557 		(*tree)->refcnt++;
558 		return NULL;
559 	}
560 
561 	return open_tree(tree, repo, id, 0);
562 }
563 
564 const struct got_error *
565 got_object_tree_open(struct got_tree_object **tree,
566     struct got_repository *repo, struct got_object *obj)
567 {
568 	return open_tree(tree, repo, got_object_get_id(obj), 1);
569 }
570 
571 const struct got_error *
572 got_object_open_as_blob(struct got_blob_object **blob,
573     struct got_repository *repo, struct got_object_id *id, size_t blocksize,
574     int outfd)
575 {
576 	return got_error(GOT_ERR_NOT_IMPL);
577 }
578 
579 const struct got_error *
580 got_object_blob_open(struct got_blob_object **blob,
581     struct got_repository *repo, struct got_object *obj, size_t blocksize,
582     int outfd)
583 {
584 	return got_error(GOT_ERR_NOT_IMPL);
585 }
586 
587 static const struct got_error *
588 open_tag(struct got_tag_object **tag, struct got_repository *repo,
589     struct got_object_id *id, int check_cache)
590 {
591 	const struct got_error *err = NULL;
592 	struct got_packidx *packidx = NULL;
593 	int idx;
594 	char *path_packfile = NULL;
595 	struct got_object *obj = NULL;
596 	int obj_type = GOT_OBJ_TYPE_ANY;
597 
598 	if (check_cache) {
599 		*tag = got_repo_get_cached_tag(repo, id);
600 		if (*tag != NULL) {
601 			(*tag)->refcnt++;
602 			return NULL;
603 		}
604 	} else
605 		*tag = NULL;
606 
607 	err = got_repo_search_packidx(&packidx, &idx, repo, id);
608 	if (err == NULL) {
609 		struct got_pack *pack = NULL;
610 		uint8_t *buf = NULL;
611 		size_t len;
612 
613 		err = got_packidx_get_packfile_path(&path_packfile,
614 		    packidx->path_packidx);
615 		if (err)
616 			return err;
617 
618 		pack = got_repo_get_cached_pack(repo, path_packfile);
619 		if (pack == NULL) {
620 			err = got_repo_cache_pack(&pack, repo, path_packfile,
621 			    packidx);
622 			if (err)
623 				goto done;
624 		}
625 
626 		/* Beware of "lightweight" tags: Check object type first. */
627 		err = got_packfile_open_object(&obj, pack, packidx, idx, id);
628 		if (err)
629 			goto done;
630 		obj_type = obj->type;
631 		if (obj_type != GOT_OBJ_TYPE_TAG) {
632 			err = got_error(GOT_ERR_OBJ_TYPE);
633 			got_object_close(obj);
634 			goto done;
635 		}
636 		err = got_packfile_extract_object_to_mem(&buf, &len,
637 		    obj, pack);
638 		got_object_close(obj);
639 		if (err)
640 			goto done;
641 		err = got_object_parse_tag(tag, buf, len);
642 		free(buf);
643 	} else if (err->code == GOT_ERR_NO_OBJ) {
644 		int fd;
645 
646 		err = got_object_open_loose_fd(&fd, id, repo);
647 		if (err)
648 			return err;
649 		err = got_object_read_header(&obj, fd);
650 		if (close(fd) == -1 && err == NULL)
651 			err = got_error_from_errno("close");
652 		if (err)
653 			return err;
654 		obj_type = obj->type;
655 		got_object_close(obj);
656 		if (obj_type != GOT_OBJ_TYPE_TAG)
657 			return got_error(GOT_ERR_OBJ_TYPE);
658 
659 		err = got_object_open_loose_fd(&fd, id, repo);
660 		if (err)
661 			return err;
662 		err = got_object_read_tag(tag, fd, id, 0);
663 		if (close(fd) == -1 && err == NULL)
664 			err = got_error_from_errno("close");
665 		if (err)
666 			return err;
667 	}
668 
669 	if (err == NULL) {
670 		(*tag)->refcnt++;
671 		err = got_repo_cache_tag(repo, id, *tag);
672 		if (err) {
673 			if (err->code == GOT_ERR_OBJ_EXISTS ||
674 			    err->code == GOT_ERR_OBJ_TOO_LARGE)
675 				err = NULL;
676 		}
677 	}
678 done:
679 	free(path_packfile);
680 	return err;
681 }
682 
683 const struct got_error *
684 got_object_open_as_tag(struct got_tag_object **tag,
685     struct got_repository *repo, struct got_object_id *id)
686 {
687 	*tag = got_repo_get_cached_tag(repo, id);
688 	if (*tag != NULL) {
689 		(*tag)->refcnt++;
690 		return NULL;
691 	}
692 
693 	return open_tag(tag, repo, id, 0);
694 }
695 
696 const struct got_error *
697 got_object_tag_open(struct got_tag_object **tag,
698     struct got_repository *repo, struct got_object *obj)
699 {
700 	return open_tag(tag, repo, got_object_get_id(obj), 1);
701 }