Got Mirror

Blob

Date:: Sun Apr 1 13:59:36 2018 UTC
Message:: remove debug code accidentally committed
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 
21 #include <dirent.h>
22 #include <fcntl.h>
23 #include <errno.h>
24 #include <stdio.h>
25 #include <stdint.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <limits.h>
29 #include <sha1.h>
30 #include <endian.h>
31 #include <zlib.h>
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_repository.h"
36 
37 #include "got_lib_sha1.h"
38 #include "got_lib_pack.h"
39 #include "got_lib_path.h"
40 #include "got_lib_delta.h"
41 #include "got_lib_zbuf.h"
42 #include "got_lib_object.h"
43 #include "got_lib_repository.h"
44 
45 #ifndef nitems
46 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
47 #endif
48 
49 #define GOT_PACK_PREFIX		"pack-"
50 #define GOT_PACKFILE_SUFFIX	".pack"
51 #define GOT_PACKIDX_SUFFIX		".idx"
52 #define GOT_PACKFILE_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
53 				SHA1_DIGEST_STRING_LENGTH - 1 + \
54 				strlen(GOT_PACKFILE_SUFFIX))
55 #define GOT_PACKIDX_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
56 				SHA1_DIGEST_STRING_LENGTH - 1 + \
57 				strlen(GOT_PACKIDX_SUFFIX))
58 
59 #ifndef MIN
60 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
61 #endif
62 
63 static const struct got_error *
64 verify_fanout_table(uint32_t *fanout_table)
65 {
66 	int i;
67 
68 	for (i = 0; i < 0xff - 1; i++) {
69 		if (be32toh(fanout_table[i]) > be32toh(fanout_table[i + 1]))
70 			return got_error(GOT_ERR_BAD_PACKIDX);
71 	}
72 
73 	return NULL;
74 }
75 
76 static const struct got_error *
77 get_packfile_size(size_t *size, const char *path_idx)
78 {
79 	struct stat sb;
80 	char *path_pack;
81 	char base_path[PATH_MAX];
82 	char *dot;
83 
84 	if (strlcpy(base_path, path_idx, PATH_MAX) > PATH_MAX)
85 		return got_error(GOT_ERR_NO_SPACE);
86 
87 	dot = strrchr(base_path, '.');
88 	if (dot == NULL)
89 		return got_error(GOT_ERR_BAD_PATH);
90 	*dot = '\0';
91 	if (asprintf(&path_pack, "%s.pack", base_path) == -1)
92 		return got_error_from_errno();
93 
94 	if (stat(path_pack, &sb) != 0) {
95 		free(path_pack);
96 		return got_error_from_errno();
97 	}
98 
99 	free(path_pack);
100 	*size = sb.st_size;
101 	return 0;
102 }
103 
104 const struct got_error *
105 got_packidx_open(struct got_packidx_v2_hdr **packidx, const char *path)
106 {
107 	struct got_packidx_v2_hdr *p;
108 	FILE *f;
109 	const struct got_error *err = NULL;
110 	size_t n, nobj, packfile_size;
111 	SHA1_CTX ctx;
112 	uint8_t sha1[SHA1_DIGEST_LENGTH];
113 
114 	SHA1Init(&ctx);
115 
116 	f = fopen(path, "rb");
117 	if (f == NULL)
118 		return got_error_from_errno();
119 
120 	err = get_packfile_size(&packfile_size, path);
121 	if (err)
122 		return err;
123 
124 	p = calloc(1, sizeof(*p));
125 	if (p == NULL) {
126 		err = got_error_from_errno();
127 		goto done;
128 	}
129 
130 	n = fread(&p->magic, sizeof(p->magic), 1, f);
131 	if (n != 1) {
132 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
133 		goto done;
134 	}
135 
136 	if (betoh32(p->magic) != GOT_PACKIDX_V2_MAGIC) {
137 		err = got_error(GOT_ERR_BAD_PACKIDX);
138 		goto done;
139 	}
140 
141 	SHA1Update(&ctx, (uint8_t *)&p->magic, sizeof(p->magic));
142 
143 	n = fread(&p->version, sizeof(p->version), 1, f);
144 	if (n != 1) {
145 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
146 		goto done;
147 	}
148 
149 	if (betoh32(p->version) != GOT_PACKIDX_VERSION) {
150 		err = got_error(GOT_ERR_BAD_PACKIDX);
151 		goto done;
152 	}
153 
154 	SHA1Update(&ctx, (uint8_t *)&p->version, sizeof(p->version));
155 
156 	n = fread(&p->fanout_table, sizeof(p->fanout_table), 1, f);
157 	if (n != 1) {
158 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
159 		goto done;
160 	}
161 
162 	err = verify_fanout_table(p->fanout_table);
163 	if (err)
164 		goto done;
165 
166 	SHA1Update(&ctx, (uint8_t *)p->fanout_table, sizeof(p->fanout_table));
167 
168 	nobj = betoh32(p->fanout_table[0xff]);
169 
170 	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
171 	if (p->sorted_ids == NULL) {
172 		err = got_error_from_errno();
173 		goto done;
174 	}
175 
176 	n = fread(p->sorted_ids, sizeof(*p->sorted_ids), nobj, f);
177 	if (n != nobj) {
178 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
179 		goto done;
180 	}
181 
182 	SHA1Update(&ctx, (uint8_t *)p->sorted_ids,
183 	    nobj * sizeof(*p->sorted_ids));
184 
185 	p->crc32 = calloc(nobj, sizeof(*p->crc32));
186 	if (p->crc32 == NULL) {
187 		err = got_error_from_errno();
188 		goto done;
189 	}
190 
191 	n = fread(p->crc32, sizeof(*p->crc32), nobj, f);
192 	if (n != nobj) {
193 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
194 		goto done;
195 	}
196 
197 	SHA1Update(&ctx, (uint8_t *)p->crc32, nobj * sizeof(*p->crc32));
198 
199 	p->offsets = calloc(nobj, sizeof(*p->offsets));
200 	if (p->offsets == NULL) {
201 		err = got_error_from_errno();
202 		goto done;
203 	}
204 
205 	n = fread(p->offsets, sizeof(*p->offsets), nobj, f);
206 	if (n != nobj) {
207 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
208 		goto done;
209 	}
210 
211 	SHA1Update(&ctx, (uint8_t *)p->offsets, nobj * sizeof(*p->offsets));
212 
213 	/* Large file offsets are contained only in files > 2GB. */
214 	if (packfile_size <= 0x80000000)
215 		goto checksum;
216 
217 	p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
218 	if (p->large_offsets == NULL) {
219 		err = got_error_from_errno();
220 		goto done;
221 	}
222 
223 	n = fread(p->large_offsets, sizeof(*p->large_offsets), nobj, f);
224 	if (n != nobj) {
225 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
226 		goto done;
227 	}
228 
229 	SHA1Update(&ctx, (uint8_t*)p->large_offsets,
230 	    nobj * sizeof(*p->large_offsets));
231 
232 checksum:
233 	n = fread(&p->trailer, sizeof(p->trailer), 1, f);
234 	if (n != 1) {
235 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
236 		goto done;
237 	}
238 
239 	SHA1Update(&ctx, p->trailer.packfile_sha1, SHA1_DIGEST_LENGTH);
240 	SHA1Final(sha1, &ctx);
241 	if (memcmp(p->trailer.packidx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0)
242 		err = got_error(GOT_ERR_PACKIDX_CSUM);
243 done:
244 	fclose(f);
245 	if (err)
246 		got_packidx_close(p);
247 	else
248 		*packidx = p;
249 	return err;
250 }
251 
252 void
253 got_packidx_close(struct got_packidx_v2_hdr *packidx)
254 {
255 	free(packidx->sorted_ids);
256 	free(packidx->offsets);
257 	free(packidx->crc32);
258 	free(packidx->large_offsets);
259 	free(packidx);
260 }
261 
262 static int
263 is_packidx_filename(const char *name, size_t len)
264 {
265 	if (len != GOT_PACKIDX_NAMELEN)
266 		return 0;
267 
268 	if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
269 		return 0;
270 
271 	if (strcmp(name + strlen(GOT_PACK_PREFIX) +
272 	    SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
273 		return 0;
274 
275 	return 1;
276 }
277 
278 static off_t
279 get_object_offset(struct got_packidx_v2_hdr *packidx, int idx)
280 {
281 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
282 	uint32_t offset = betoh32(packidx->offsets[idx]);
283 	if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
284 		uint64_t loffset;
285 		idx = offset & GOT_PACKIDX_OFFSET_VAL_MASK;
286 		if (idx < 0 || idx > totobj || packidx->large_offsets == NULL)
287 			return -1;
288 		loffset = betoh64(packidx->large_offsets[idx]);
289 		return (loffset > INT64_MAX ? -1 : (off_t)loffset);
290 	}
291 	return (off_t)(offset & GOT_PACKIDX_OFFSET_VAL_MASK);
292 }
293 
294 static const struct got_error *
295 get_packfile_path(char **path_packfile, struct got_repository *repo,
296     struct got_packidx_v2_hdr *packidx);
297 
298 static int
299 get_object_idx(struct got_packidx_v2_hdr *packidx, struct got_object_id *id, struct got_repository *repo)
300 {
301 	u_int8_t id0 = id->sha1[0];
302 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
303 	int i = 0;
304 
305 	if (id0 > 0)
306 		i = betoh32(packidx->fanout_table[id0 - 1]);
307 
308 	while (i < totobj) {
309 		struct got_object_id *oid = &packidx->sorted_ids[i];
310 		int cmp = got_object_id_cmp(id, oid);
311 
312 		if (cmp == 0)
313 			return i;
314 		i++;
315 	}
316 
317 	return -1;
318 }
319 
320 static struct got_packidx_v2_hdr *
321 dup_packidx(struct got_packidx_v2_hdr *packidx)
322 {
323 	struct got_packidx_v2_hdr *p;
324 	size_t nobj;
325 
326 	p = calloc(1, sizeof(*p));
327 	if (p == NULL)
328 		return NULL;
329 
330 	memcpy(p, packidx, sizeof(*p));
331 	p->sorted_ids = NULL;
332 	p->crc32 = NULL;
333 	p->offsets = NULL;
334 	p->large_offsets = NULL;
335 
336 	nobj = betoh32(p->fanout_table[0xff]);
337 
338 	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
339 	if (p->sorted_ids == NULL)
340 		goto err;
341 	memcpy(p->sorted_ids, packidx->sorted_ids, nobj * sizeof(*p->sorted_ids));
342 
343 	p->crc32 = calloc(nobj, sizeof(*p->crc32));
344 	if (p->crc32 == NULL)
345 		goto err;
346 	memcpy(p->crc32, packidx->crc32, nobj * sizeof(*p->crc32));
347 
348 	p->offsets = calloc(nobj, sizeof(*p->offsets));
349 	if (p->offsets == NULL)
350 		goto err;
351 	memcpy(p->offsets, packidx->offsets, nobj * sizeof(*p->offsets));
352 
353 	if (p->large_offsets) {
354 		p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
355 		if (p->large_offsets == NULL)
356 			goto err;
357 		memcpy(p->large_offsets, packidx->large_offsets,
358 		    nobj * sizeof(*p->large_offsets));
359 	}
360 
361 	return p;
362 
363 err:
364 	free(p->large_offsets);
365 	free(p->offsets);
366 	free(p->crc32);
367 	free(p->sorted_ids);
368 	free(p);
369 	return NULL;
370 }
371 
372 static void
373 cache_packidx(struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
374 {
375 	int i;
376 
377 	for (i = 0; i < nitems(repo->packidx_cache); i++) {
378 		if (repo->packidx_cache[i] == NULL)
379 			break;
380 	}
381 
382 	if (i == nitems(repo->packidx_cache)) {
383 		got_packidx_close(repo->packidx_cache[i - 1]);
384 		memmove(&repo->packidx_cache[1], &repo->packidx_cache[0],
385 		    sizeof(repo->packidx_cache) -
386 		    sizeof(repo->packidx_cache[0]));
387 		i = 0;
388 	}
389 
390 	repo->packidx_cache[i] = dup_packidx(packidx);
391 }
392 
393 static const struct got_error *
394 search_packidx(struct got_packidx_v2_hdr **packidx, int *idx,
395     struct got_repository *repo, struct got_object_id *id)
396 {
397 	const struct got_error *err;
398 	char *path_packdir;
399 	DIR *packdir;
400 	struct dirent *dent;
401 	char *path_packidx;
402 	int i;
403 
404 	/* Search pack index cache. */
405 	for (i = 0; i < nitems(repo->packidx_cache); i++) {
406 		if (repo->packidx_cache[i] == NULL)
407 			break;
408 		*idx = get_object_idx(repo->packidx_cache[i], id, repo);
409 		if (*idx != -1) {
410 			*packidx = repo->packidx_cache[i];
411 			return NULL;
412 		}
413 	}
414 	/* No luck. Search the filesystem. */
415 
416 	path_packdir = got_repo_get_path_objects_pack(repo);
417 	if (path_packdir == NULL)
418 		return got_error_from_errno();
419 
420 	packdir = opendir(path_packdir);
421 	if (packdir == NULL) {
422 		err = got_error_from_errno();
423 		goto done;
424 	}
425 
426 	while ((dent = readdir(packdir)) != NULL) {
427 		if (!is_packidx_filename(dent->d_name, dent->d_namlen))
428 			continue;
429 
430 		if (asprintf(&path_packidx, "%s/%s", path_packdir,
431 		    dent->d_name) == -1) {
432 			err = got_error_from_errno();
433 			goto done;
434 		}
435 
436 		err = got_packidx_open(packidx, path_packidx);
437 		free(path_packidx);
438 		if (err)
439 			goto done;
440 
441 		*idx = get_object_idx(*packidx, id, repo);
442 		if (*idx != -1) {
443 			err = NULL; /* found the object */
444 			cache_packidx(*packidx, repo);
445 			goto done;
446 		}
447 
448 		got_packidx_close(*packidx);
449 		*packidx = NULL;
450 	}
451 
452 	err = got_error(GOT_ERR_NO_OBJ);
453 done:
454 	free(path_packdir);
455 	if (packdir && closedir(packdir) != 0 && err == 0)
456 		err = got_error_from_errno();
457 	return err;
458 }
459 
460 static const struct got_error *
461 get_packfile_path(char **path_packfile, struct got_repository *repo,
462     struct got_packidx_v2_hdr *packidx)
463 {
464 	char *path_packdir;
465 	char hex[SHA1_DIGEST_STRING_LENGTH];
466 	char *sha1str;
467 
468 	path_packdir = got_repo_get_path_objects_pack(repo);
469 	if (path_packdir == NULL)
470 		return got_error_from_errno();
471 
472 	sha1str = got_sha1_digest_to_str(packidx->trailer.packfile_sha1,
473 	    hex, sizeof(hex));
474 	if (sha1str == NULL)
475 		return got_error(GOT_ERR_PACKIDX_CSUM);
476 
477 	if (asprintf(path_packfile, "%s/%s%s%s", path_packdir,
478 	    GOT_PACK_PREFIX, sha1str, GOT_PACKFILE_SUFFIX) == -1) {
479 		*path_packfile = NULL;
480 		return got_error_from_errno();
481 	}
482 
483 	return NULL;
484 }
485 
486 const struct got_error *
487 read_packfile_hdr(FILE *f, struct got_packidx_v2_hdr *packidx)
488 {
489 	const struct got_error *err = NULL;
490 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
491 	struct got_packfile_hdr hdr;
492 	size_t n;
493 
494 	n = fread(&hdr, sizeof(hdr), 1, f);
495 	if (n != 1)
496 		return got_ferror(f, GOT_ERR_BAD_PACKIDX);
497 
498 	if (betoh32(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
499 	    betoh32(hdr.version) != GOT_PACKFILE_VERSION ||
500 	    betoh32(hdr.nobjects) != totobj)
501 		err = got_error(GOT_ERR_BAD_PACKFILE);
502 
503 	return err;
504 }
505 
506 static const struct got_error *
507 open_packfile(FILE **packfile, const char *path_packfile,
508     struct got_repository *repo, struct got_packidx_v2_hdr *packidx)
509 {
510 	const struct got_error *err = NULL;
511 
512 	*packfile = NULL;
513 
514 	*packfile = fopen(path_packfile, "rb");
515 	if (*packfile == NULL) {
516 		err = got_error_from_errno();
517 		return err;
518 	}
519 
520 	if (packidx) {
521 		err = read_packfile_hdr(*packfile, packidx);
522 		if (err) {
523 			fclose(*packfile);
524 			*packfile = NULL;
525 		}
526 	}
527 	return err;
528 }
529 
530 void
531 got_pack_close(struct got_pack *pack)
532 {
533 	fclose(pack->packfile);
534 	pack->packfile = NULL;
535 	free(pack->path_packfile);
536 	pack->path_packfile = NULL;
537 	pack->filesize = 0;
538 }
539 
540 static const struct got_error *
541 cache_pack(struct got_pack **packp, const char *path_packfile,
542     struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
543 {
544 	const struct got_error *err = NULL;
545 	struct got_pack *pack = NULL;
546 	int i;
547 
548 	if (packp)
549 		*packp = NULL;
550 
551 	for (i = 0; i < nitems(repo->packs); i++) {
552 		pack = &repo->packs[i];
553 		if (pack->path_packfile == NULL)
554 			break;
555 		if (strcmp(pack->path_packfile, path_packfile) == 0)
556 			return NULL;
557 	}
558 
559 	if (i == nitems(repo->packs) - 1) {
560 		got_pack_close(&repo->packs[i - 1]);
561 		memmove(&repo->packs[1], &repo->packs[0],
562 		    sizeof(repo->packs) - sizeof(repo->packs[0]));
563 		i = 0;
564 	}
565 
566 	pack = &repo->packs[i];
567 
568 	pack->path_packfile = strdup(path_packfile);
569 	if (pack->path_packfile == NULL) {
570 		err = got_error_from_errno();
571 		goto done;
572 	}
573 
574 	err = open_packfile(&pack->packfile, path_packfile, repo, packidx);
575 	if (err)
576 		goto done;
577 
578 	err = get_packfile_size(&pack->filesize, path_packfile);
579 done:
580 	if (err) {
581 		if (pack)
582 			free(pack->path_packfile);
583 		free(pack);
584 	} else if (packp)
585 		*packp = pack;
586 	return err;
587 }
588 
589 struct got_pack *
590 get_cached_pack(const char *path_packfile, struct got_repository *repo)
591 {
592 	struct got_pack *pack = NULL;
593 	int i;
594 
595 	for (i = 0; i < nitems(repo->packs); i++) {
596 		pack = &repo->packs[i];
597 		if (pack->path_packfile == NULL)
598 			break;
599 		if (strcmp(pack->path_packfile, path_packfile) == 0)
600 			return pack;
601 	}
602 
603 	return NULL;
604 }
605 
606 static const struct got_error *
607 parse_object_type_and_size(uint8_t *type, uint64_t *size, size_t *len,
608     FILE *packfile)
609 {
610 	uint8_t t = 0;
611 	uint64_t s = 0;
612 	uint8_t sizeN;
613 	size_t n;
614 	int i = 0;
615 
616 	do {
617 		/* We do not support size values which don't fit in 64 bit. */
618 		if (i > 9)
619 			return got_error(GOT_ERR_NO_SPACE);
620 
621 		n = fread(&sizeN, sizeof(sizeN), 1, packfile);
622 		if (n != 1)
623 			return got_ferror(packfile, GOT_ERR_BAD_PACKIDX);
624 
625 		if (i == 0) {
626 			t = (sizeN & GOT_PACK_OBJ_SIZE0_TYPE_MASK) >>
627 			    GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT;
628 			s = (sizeN & GOT_PACK_OBJ_SIZE0_VAL_MASK);
629 		} else {
630 			size_t shift = 4 + 7 * (i - 1);
631 			s |= ((sizeN & GOT_PACK_OBJ_SIZE_VAL_MASK) << shift);
632 		}
633 		i++;
634 	} while (sizeN & GOT_PACK_OBJ_SIZE_MORE);
635 
636 	*type = t;
637 	*size = s;
638 	*len = i * sizeof(sizeN);
639 	return NULL;
640 }
641 
642 static const struct got_error *
643 open_plain_object(struct got_object **obj, const char *path_packfile,
644     struct got_object_id *id, uint8_t type, off_t offset, size_t size)
645 {
646 	*obj = calloc(1, sizeof(**obj));
647 	if (*obj == NULL)
648 		return got_error_from_errno();
649 
650 	(*obj)->path_packfile = strdup(path_packfile);
651 	if ((*obj)->path_packfile == NULL) {
652 		const struct got_error *err = got_error_from_errno();
653 		free(*obj);
654 		*obj = NULL;
655 		return err;
656 	}
657 
658 	(*obj)->type = type;
659 	(*obj)->flags = GOT_OBJ_FLAG_PACKED;
660 	(*obj)->hdrlen = 0;
661 	(*obj)->size = size;
662 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
663 	(*obj)->pack_offset = offset;
664 
665 	return NULL;
666 }
667 
668 static const struct got_error *
669 parse_negative_offset(int64_t *offset, size_t *len, FILE *packfile)
670 {
671 	int64_t o = 0;
672 	uint8_t offN;
673 	size_t n;
674 	int i = 0;
675 
676 	do {
677 		/* We do not support offset values which don't fit in 64 bit. */
678 		if (i > 8)
679 			return got_error(GOT_ERR_NO_SPACE);
680 
681 		n = fread(&offN, sizeof(offN), 1, packfile);
682 		if (n != 1)
683 			return got_ferror(packfile, GOT_ERR_BAD_PACKIDX);
684 
685 		if (i == 0)
686 			o = (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
687 		else {
688 			o++;
689 			o <<= 7;
690 			o += (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
691 		}
692 		i++;
693 	} while (offN & GOT_PACK_OBJ_DELTA_OFF_MORE);
694 
695 	*offset = o;
696 	*len = i * sizeof(offN);
697 	return NULL;
698 }
699 
700 static const struct got_error *
701 parse_offset_delta(off_t *base_offset, FILE *packfile, off_t offset)
702 {
703 	const struct got_error *err;
704 	int64_t negoffset;
705 	size_t negofflen;
706 
707 	err = parse_negative_offset(&negoffset, &negofflen, packfile);
708 	if (err)
709 		return err;
710 
711 	/* Compute the base object's offset (must be in the same pack file). */
712 	*base_offset = (offset - negoffset);
713 	if (*base_offset <= 0)
714 		return got_error(GOT_ERR_BAD_PACKFILE);
715 
716 	return NULL;
717 }
718 
719 static const struct got_error *
720 resolve_delta_chain(struct got_delta_chain *, struct got_repository *,
721     FILE *, const char *, off_t, size_t, int, size_t);
722 
723 static const struct got_error *
724 add_delta(struct got_delta_chain *deltas, const char *path_packfile,
725     off_t delta_offset, size_t tslen, int delta_type, size_t delta_size,
726     size_t delta_data_offset, uint8_t *delta_buf, size_t delta_len)
727 {
728 	struct got_delta *delta;
729 
730 	delta = got_delta_open(path_packfile, delta_offset, tslen,
731 	    delta_type, delta_size, delta_data_offset, delta_buf,
732 	    delta_len);
733 	if (delta == NULL)
734 		return got_error_from_errno();
735 	/* delta is freed in got_object_close() */
736 	deltas->nentries++;
737 	SIMPLEQ_INSERT_HEAD(&deltas->entries, delta, entry);
738 	return NULL;
739 }
740 
741 static const struct got_error *
742 resolve_offset_delta(struct got_delta_chain *deltas,
743     struct got_repository *repo, FILE *packfile, const char *path_packfile,
744     off_t delta_offset,size_t tslen, int delta_type, size_t delta_size)
745 
746 {
747 	const struct got_error *err;
748 	off_t base_offset;
749 	uint8_t base_type;
750 	uint64_t base_size;
751 	size_t base_tslen;
752 	off_t delta_data_offset;
753 	uint8_t *delta_buf;
754 	size_t delta_len;
755 
756 	err = parse_offset_delta(&base_offset, packfile, delta_offset);
757 	if (err)
758 		return err;
759 
760 	delta_data_offset = ftello(packfile);
761 	if (delta_data_offset == -1)
762 		return got_error_from_errno();
763 
764 	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
765 	if (err)
766 		return err;
767 
768 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
769 	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
770 	if (err)
771 		return err;
772 
773 	/* An offset delta must be in the same packfile. */
774 	if (fseeko(packfile, base_offset, SEEK_SET) != 0)
775 		return got_error_from_errno();
776 
777 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
778 	    packfile);
779 	if (err)
780 		return err;
781 
782 	return resolve_delta_chain(deltas, repo, packfile, path_packfile,
783 	    base_offset, base_tslen, base_type, base_size);
784 }
785 
786 static const struct got_error *
787 resolve_ref_delta(struct got_delta_chain *deltas, struct got_repository *repo,
788     FILE *packfile, const char *path_packfile, off_t delta_offset,
789     size_t tslen, int delta_type, size_t delta_size)
790 {
791 	const struct got_error *err;
792 	struct got_object_id id;
793 	struct got_packidx_v2_hdr *packidx;
794 	int idx;
795 	off_t base_offset;
796 	uint8_t base_type;
797 	uint64_t base_size;
798 	size_t base_tslen;
799 	size_t n;
800 	char *path_base_packfile;
801 	struct got_pack *base_pack;
802 	off_t delta_data_offset;
803 	uint8_t *delta_buf;
804 	size_t delta_len;
805 
806 	n = fread(&id, sizeof(id), 1, packfile);
807 	if (n != 1)
808 		return got_ferror(packfile, GOT_ERR_IO);
809 
810 	delta_data_offset = ftello(packfile);
811 	if (delta_data_offset == -1)
812 		return got_error_from_errno();
813 
814 	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
815 	if (err)
816 		return err;
817 
818 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
819 	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
820 	if (err)
821 		return err;
822 
823 	err = search_packidx(&packidx, &idx, repo, &id);
824 	if (err)
825 		return err;
826 
827 	base_offset = get_object_offset(packidx, idx);
828 	if (base_offset == (uint64_t)-1) {
829 		return got_error(GOT_ERR_BAD_PACKIDX);
830 	}
831 
832 	err = get_packfile_path(&path_base_packfile, repo, packidx);
833 	if (err)
834 		return err;
835 
836 	base_pack = get_cached_pack(path_base_packfile, repo);
837 	if (base_pack == NULL) {
838 		err = cache_pack(&base_pack, path_base_packfile, NULL, repo);
839 		if (err)
840 			goto done;
841 	}
842 
843 	if (fseeko(base_pack->packfile, base_offset, SEEK_SET) != 0) {
844 		err = got_error_from_errno();
845 		goto done;
846 	}
847 
848 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
849 	    base_pack->packfile);
850 	if (err)
851 		goto done;
852 
853 	err = resolve_delta_chain(deltas, repo, base_pack->packfile,
854 	    path_base_packfile, base_offset, base_tslen, base_type,
855 	    base_size);
856 done:
857 	free(path_base_packfile);
858 	return err;
859 }
860 
861 static const struct got_error *
862 resolve_delta_chain(struct got_delta_chain *deltas, struct got_repository *repo,
863     FILE *packfile, const char *path_packfile, off_t delta_offset, size_t tslen,
864     int delta_type, size_t delta_size)
865 {
866 	const struct got_error *err = NULL;
867 
868 	switch (delta_type) {
869 	case GOT_OBJ_TYPE_COMMIT:
870 	case GOT_OBJ_TYPE_TREE:
871 	case GOT_OBJ_TYPE_BLOB:
872 	case GOT_OBJ_TYPE_TAG:
873 		/* Plain types are the final delta base. Recursion ends. */
874 		err = add_delta(deltas, path_packfile, delta_offset, tslen,
875 		    delta_type, delta_size, 0, NULL, 0);
876 		break;
877 	case GOT_OBJ_TYPE_OFFSET_DELTA:
878 		err = resolve_offset_delta(deltas, repo, packfile,
879 		    path_packfile, delta_offset, tslen, delta_type,
880 		    delta_size);
881 		break;
882 	case GOT_OBJ_TYPE_REF_DELTA:
883 		err = resolve_ref_delta(deltas, repo, packfile,
884 		    path_packfile, delta_offset, tslen, delta_type,
885 		    delta_size);
886 		break;
887 	default:
888 		return got_error(GOT_ERR_NOT_IMPL);
889 	}
890 
891 	return err;
892 }
893 
894 static const struct got_error *
895 open_delta_object(struct got_object **obj, struct got_repository *repo,
896     struct got_packidx_v2_hdr *packidx, const char *path_packfile,
897     FILE *packfile, struct got_object_id *id, off_t offset, size_t tslen,
898     int delta_type, size_t delta_size)
899 {
900 	const struct got_error *err = NULL;
901 	int resolved_type;
902 
903 	*obj = calloc(1, sizeof(**obj));
904 	if (*obj == NULL)
905 		return got_error_from_errno();
906 
907 	(*obj)->flags = 0;
908 	(*obj)->hdrlen = 0;
909 	(*obj)->size = 0; /* Not known because deltas aren't applied yet. */
910 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
911 	(*obj)->pack_offset = offset + tslen;
912 
913 	(*obj)->path_packfile = strdup(path_packfile);
914 	if ((*obj)->path_packfile == NULL) {
915 		err = got_error_from_errno();
916 		goto done;
917 	}
918 	(*obj)->flags |= GOT_OBJ_FLAG_PACKED;
919 
920 	SIMPLEQ_INIT(&(*obj)->deltas.entries);
921 	(*obj)->flags |= GOT_OBJ_FLAG_DELTIFIED;
922 
923 	err = resolve_delta_chain(&(*obj)->deltas, repo, packfile,
924 	    path_packfile, offset, tslen, delta_type, delta_size);
925 	if (err)
926 		goto done;
927 
928 	err = got_delta_chain_get_base_type(&resolved_type, &(*obj)->deltas);
929 	if (err)
930 		goto done;
931 	(*obj)->type = resolved_type;
932 
933 done:
934 	if (err) {
935 		got_object_close(*obj);
936 		*obj = NULL;
937 	}
938 	return err;
939 }
940 
941 static const struct got_error *
942 open_packed_object(struct got_object **obj, struct got_repository *repo,
943     struct got_packidx_v2_hdr *packidx, int idx, struct got_object_id *id)
944 {
945 	const struct got_error *err = NULL;
946 	off_t offset;
947 	char *path_packfile;
948 	struct got_pack *pack;
949 	uint8_t type;
950 	uint64_t size;
951 	size_t tslen;
952 
953 	*obj = NULL;
954 
955 	offset = get_object_offset(packidx, idx);
956 	if (offset == (uint64_t)-1)
957 		return got_error(GOT_ERR_BAD_PACKIDX);
958 
959 	err = get_packfile_path(&path_packfile, repo, packidx);
960 	if (err)
961 		return err;
962 
963 	pack = get_cached_pack(path_packfile, repo);
964 	if (pack == NULL) {
965 		err = cache_pack(&pack, path_packfile, packidx, repo);
966 		if (err)
967 			goto done;
968 	}
969 
970 	if (fseeko(pack->packfile, offset, SEEK_SET) != 0) {
971 		err = got_error_from_errno();
972 		goto done;
973 	}
974 
975 	err = parse_object_type_and_size(&type, &size, &tslen, pack->packfile);
976 	if (err)
977 		goto done;
978 
979 	switch (type) {
980 	case GOT_OBJ_TYPE_COMMIT:
981 	case GOT_OBJ_TYPE_TREE:
982 	case GOT_OBJ_TYPE_BLOB:
983 	case GOT_OBJ_TYPE_TAG:
984 		err = open_plain_object(obj, path_packfile, id, type,
985 		    offset + tslen, size);
986 		break;
987 
988 	case GOT_OBJ_TYPE_OFFSET_DELTA:
989 	case GOT_OBJ_TYPE_REF_DELTA:
990 		err = open_delta_object(obj, repo, packidx, path_packfile,
991 		    pack->packfile, id, offset, tslen, type, size);
992 		break;
993 
994 	default:
995 		err = got_error(GOT_ERR_NOT_IMPL);
996 		goto done;
997 	}
998 done:
999 	free(path_packfile);
1000 	return err;
1001 }
1002 
1003 const struct got_error *
1004 got_packfile_open_object(struct got_object **obj, struct got_object_id *id,
1005     struct got_repository *repo)
1006 {
1007 	const struct got_error *err = NULL;
1008 	struct got_packidx_v2_hdr *packidx = NULL;
1009 	int idx;
1010 
1011 	err = search_packidx(&packidx, &idx, repo, id);
1012 	if (err)
1013 		return err;
1014 
1015 	err = open_packed_object(obj, repo, packidx, idx, id);
1016 	if (err)
1017 		return err;
1018 
1019 	err = cache_pack(NULL, (*obj)->path_packfile, packidx, repo);
1020 	return err;
1021 }
1022 
1023 static const struct got_error *
1024 get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas)
1025 {
1026 	struct got_delta *delta;
1027 	uint64_t base_size = 0, result_size = 0;
1028 
1029 	*max_size = 0;
1030 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1031 		/* Plain object types are the delta base. */
1032 		if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1033 		    delta->type != GOT_OBJ_TYPE_TREE &&
1034 		    delta->type != GOT_OBJ_TYPE_BLOB &&
1035 		    delta->type != GOT_OBJ_TYPE_TAG) {
1036 			const struct got_error *err;
1037 			err = got_delta_get_sizes(&base_size, &result_size,
1038 			    delta->delta_buf, delta->delta_len);
1039 			if (err)
1040 				return err;
1041 		} else
1042 			base_size = delta->size;
1043 		if (base_size > *max_size)
1044 			*max_size = base_size;
1045 		if (result_size > *max_size)
1046 			*max_size = result_size;
1047 	}
1048 
1049 	return NULL;
1050 }
1051 
1052 static const struct got_error *
1053 dump_delta_chain_to_file(size_t *result_size, struct got_delta_chain *deltas,
1054     FILE *outfile, struct got_repository *repo)
1055 {
1056 	const struct got_error *err = NULL;
1057 	struct got_delta *delta;
1058 	FILE *base_file = NULL, *accum_file = NULL;
1059 	uint8_t *base_buf = NULL, *accum_buf = NULL;
1060 	size_t accum_size = 0;
1061 	uint64_t max_size;
1062 	int n = 0;
1063 
1064 	*result_size = 0;
1065 
1066 	if (SIMPLEQ_EMPTY(&deltas->entries))
1067 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
1068 
1069 	/* We process small enough files entirely in memory for speed. */
1070 	err = get_delta_chain_max_size(&max_size, deltas);
1071 	if (err)
1072 		return err;
1073 	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
1074 		accum_buf = malloc(max_size);
1075 		if (accum_buf == NULL)
1076 			return got_error_from_errno();
1077 	} else {
1078 		base_file = got_opentemp();
1079 		if (base_file == NULL)
1080 			return got_error_from_errno();
1081 
1082 		accum_file = got_opentemp();
1083 		if (accum_file == NULL) {
1084 			err = got_error_from_errno();
1085 			fclose(base_file);
1086 			return err;
1087 		}
1088 	}
1089 
1090 	/* Deltas are ordered in ascending order. */
1091 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1092 		if (n == 0) {
1093 			struct got_pack *pack;
1094 			size_t base_len;
1095 
1096 			/* Plain object types are the delta base. */
1097 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1098 			    delta->type != GOT_OBJ_TYPE_TREE &&
1099 			    delta->type != GOT_OBJ_TYPE_BLOB &&
1100 			    delta->type != GOT_OBJ_TYPE_TAG) {
1101 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1102 				goto done;
1103 			}
1104 
1105 			pack = get_cached_pack(delta->path_packfile, repo);
1106 			if (pack == NULL) {
1107 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1108 				goto done;
1109 			}
1110 
1111 			if (fseeko(pack->packfile, delta->offset + delta->tslen,
1112 			    SEEK_SET) != 0) {
1113 				err = got_error_from_errno();
1114 				goto done;
1115 			}
1116 			if (base_file)
1117 				err = got_inflate_to_file(&base_len,
1118 				    pack->packfile, base_file);
1119 			else {
1120 				err = got_inflate_to_mem(&base_buf, &base_len,
1121 				    pack->packfile);
1122 				if (base_len < max_size) {
1123 					uint8_t *p;
1124 					p = reallocarray(base_buf, 1, max_size);
1125 					if (p == NULL) {
1126 						err = got_error_from_errno();
1127 						goto done;
1128 					}
1129 					base_buf = p;
1130 				}
1131 			}
1132 			if (err)
1133 				goto done;
1134 			n++;
1135 			if (base_file)
1136 				rewind(base_file);
1137 			continue;
1138 		}
1139 
1140 		if (base_buf) {
1141 			err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
1142 			    delta->delta_len, accum_buf, &accum_size);
1143 			n++;
1144 		} else {
1145 			err = got_delta_apply(base_file, delta->delta_buf,
1146 			    delta->delta_len,
1147 			    /* Final delta application writes to output file. */
1148 			    ++n < deltas->nentries ? accum_file : outfile,
1149 			    &accum_size);
1150 		}
1151 		if (err)
1152 			goto done;
1153 
1154 		if (n < deltas->nentries) {
1155 			/* Accumulated delta becomes the new base. */
1156 			if (base_buf) {
1157 				uint8_t *tmp = accum_buf;
1158 				accum_buf = base_buf;
1159 				base_buf = tmp;
1160 			} else {
1161 				FILE *tmp = accum_file;
1162 				accum_file = base_file;
1163 				base_file = tmp;
1164 				rewind(base_file);
1165 				rewind(accum_file);
1166 			}
1167 		}
1168 	}
1169 
1170 done:
1171 	free(base_buf);
1172 	if (accum_buf) {
1173 		size_t len = fwrite(accum_buf, 1, accum_size, outfile);
1174 		free(accum_buf);
1175 		if (len != accum_size)
1176 			return got_ferror(outfile, GOT_ERR_IO);
1177 	}
1178 	if (base_file)
1179 		fclose(base_file);
1180 	if (accum_file)
1181 		fclose(accum_file);
1182 	rewind(outfile);
1183 	if (err == NULL)
1184 		*result_size = accum_size;
1185 	return err;
1186 }
1187 
1188 static const struct got_error *
1189 dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outlen,
1190     struct got_delta_chain *deltas, struct got_repository *repo)
1191 {
1192 	const struct got_error *err = NULL;
1193 	struct got_delta *delta;
1194 	uint8_t *base_buf = NULL, *accum_buf = NULL;
1195 	size_t accum_size;
1196 	uint64_t max_size;
1197 	int n = 0;
1198 
1199 	*outbuf = NULL;
1200 	*outlen = 0;
1201 
1202 	if (SIMPLEQ_EMPTY(&deltas->entries))
1203 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
1204 
1205 	err = get_delta_chain_max_size(&max_size, deltas);
1206 	if (err)
1207 		return err;
1208 	accum_buf = malloc(max_size);
1209 	if (accum_buf == NULL)
1210 		return got_error_from_errno();
1211 
1212 	/* Deltas are ordered in ascending order. */
1213 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1214 		if (n == 0) {
1215 			struct got_pack *pack;
1216 			size_t base_len;
1217 
1218 			/* Plain object types are the delta base. */
1219 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1220 			    delta->type != GOT_OBJ_TYPE_TREE &&
1221 			    delta->type != GOT_OBJ_TYPE_BLOB &&
1222 			    delta->type != GOT_OBJ_TYPE_TAG) {
1223 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1224 				goto done;
1225 			}
1226 
1227 			pack = get_cached_pack(delta->path_packfile, repo);
1228 			if (pack == NULL) {
1229 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1230 				goto done;
1231 			}
1232 
1233 			if (fseeko(pack->packfile, delta->offset + delta->tslen,
1234 			    SEEK_SET) != 0) {
1235 				err = got_error_from_errno();
1236 				goto done;
1237 			}
1238 			err = got_inflate_to_mem(&base_buf, &base_len,
1239 			    pack->packfile);
1240 			if (base_len < max_size) {
1241 				uint8_t *p;
1242 				p = reallocarray(base_buf, 1, max_size);
1243 				if (p == NULL) {
1244 					err = got_error_from_errno();
1245 					goto done;
1246 				}
1247 				base_buf = p;
1248 			}
1249 			if (err)
1250 				goto done;
1251 			n++;
1252 			continue;
1253 		}
1254 
1255 		err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
1256 		    delta->delta_len, accum_buf, &accum_size);
1257 		n++;
1258 		if (err)
1259 			goto done;
1260 
1261 		if (n < deltas->nentries) {
1262 			/* Accumulated delta becomes the new base. */
1263 			uint8_t *tmp = accum_buf;
1264 			accum_buf = base_buf;
1265 			base_buf = tmp;
1266 		}
1267 	}
1268 
1269 done:
1270 	free(base_buf);
1271 	if (err) {
1272 		free(accum_buf);
1273 		*outbuf = NULL;
1274 		*outlen = 0;
1275 	} else {
1276 		*outbuf = accum_buf;
1277 		*outlen = accum_size;
1278 	}
1279 	return err;
1280 }
1281 
1282 const struct got_error *
1283 got_packfile_extract_object(FILE **f, struct got_object *obj,
1284     struct got_repository *repo)
1285 {
1286 	const struct got_error *err = NULL;
1287 
1288 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1289 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1290 
1291 	*f = got_opentemp();
1292 	if (*f == NULL) {
1293 		err = got_error(GOT_ERR_FILE_OPEN);
1294 		goto done;
1295 	}
1296 
1297 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1298 		struct got_pack *pack;
1299 
1300 		pack = get_cached_pack(obj->path_packfile, repo);
1301 		if (pack == NULL) {
1302 			err = cache_pack(&pack, obj->path_packfile, NULL, repo);
1303 			if (err)
1304 				goto done;
1305 		}
1306 
1307 		if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
1308 			err = got_error_from_errno();
1309 			goto done;
1310 		}
1311 
1312 		err = got_inflate_to_file(&obj->size, pack->packfile, *f);
1313 	} else
1314 		err = dump_delta_chain_to_file(&obj->size, &obj->deltas, *f,
1315 		    repo);
1316 done:
1317 	if (err && *f) {
1318 		fclose(*f);
1319 		*f = NULL;
1320 	}
1321 	return err;
1322 }
1323 
1324 const struct got_error *
1325 got_packfile_extract_object_to_mem(uint8_t **buf, size_t *len,
1326     struct got_object *obj, struct got_repository *repo)
1327 {
1328 	const struct got_error *err = NULL;
1329 
1330 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1331 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1332 
1333 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1334 		struct got_pack *pack;
1335 
1336 		pack = get_cached_pack(obj->path_packfile, repo);
1337 		if (pack == NULL) {
1338 			err = cache_pack(&pack, obj->path_packfile, NULL, repo);
1339 			if (err)
1340 				goto done;
1341 		}
1342 
1343 		if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
1344 			err = got_error_from_errno();
1345 			goto done;
1346 		}
1347 
1348 		err = got_inflate_to_mem(buf, len, pack->packfile);
1349 	} else
1350 		err = dump_delta_chain_to_mem(buf, len, &obj->deltas, repo);
1351 done:
1352 	return err;
1353 }