1 /*
2  * Copyright (c) 2023 Omar Polo <op@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include "got_compat.h"
18 
19 #include <sys/queue.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/time.h>
23 #include <sys/tree.h>
24 #include <sys/types.h>
25 #include <sys/uio.h>
26 #include <sys/wait.h>
27 
28 #include <endian.h>
29 #include <limits.h>
30 #include <sha1.h>
31 #include <sha2.h>
32 #include <stdint.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <unistd.h>
37 #include <imsg.h>
38 
39 #include "got_error.h"
40 #include "got_cancel.h"
41 #include "got_object.h"
42 #include "got_opentemp.h"
43 #include "got_path.h"
44 #include "got_reference.h"
45 #include "got_repository.h"
46 #include "got_repository_load.h"
47 
48 #include "got_lib_delta.h"
49 #include "got_lib_hash.h"
50 #include "got_lib_object.h"
51 #include "got_lib_object_cache.h"
52 #include "got_lib_pack.h"
53 #include "got_lib_ratelimit.h"
54 #include "got_lib_repository.h"
55 #include "got_lib_privsep.h"
56 
57 #define GIT_BUNDLE_SIGNATURE_V2 "# v2 git bundle\n"
58 
59 #ifndef nitems
60 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
61 #endif
62 
63 #ifndef ssizeof
64 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
65 #endif
66 
67 static const struct got_error *
68 temp_file(int *fd, char **path, const char *ext, struct got_repository *repo)
69 {
70 	const struct got_error *err;
71 	char p[PATH_MAX];
72 	int r;
73 
74 	*path = NULL;
75 
76 	r = snprintf(p, sizeof(p), "%s/%s/loading",
77 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR);
78 	if (r < 0 || (size_t)r >= sizeof(p))
79 		return got_error_from_errno("snprintf");
80 
81 	err = got_opentemp_named_fd(path, fd, p, ext);
82 	if (err)
83 		return err;
84 
85 	if (fchmod(*fd, GOT_DEFAULT_FILE_MODE) == -1)
86 		return got_error_from_errno("fchmod");
87 
88 	return NULL;
89 }
90 
91 static const struct got_error *
92 load_report_progress(got_load_progress_cb progress_cb, void *progress_arg,
93     struct got_ratelimit *rl, off_t packsiz, int nobj_total,
94     int nobj_indexed, int nobj_loose, int nobj_resolved)
95 {
96 	const struct got_error *err;
97 	int elapsed;
98 
99 	if (progress_cb == NULL)
100 		return NULL;
101 
102 	err = got_ratelimit_check(&elapsed, rl);
103 	if (err || !elapsed)
104 		return err;
105 
106 	return progress_cb(progress_arg, packsiz, nobj_total, nobj_indexed,
107 	    nobj_loose, nobj_resolved);
108 }
109 
110 static const struct got_error *
111 copypack(FILE *in, int outfd, off_t *tot,
112     struct got_object_id *id, struct got_ratelimit *rl,
113     got_load_progress_cb progress_cb, void *progress_arg,
114     got_cancel_cb cancel_cb, void *cancel_arg)
115 {
116 	const struct got_error *err;
117 	struct got_hash hash;
118 	struct got_object_id expected_id;
119 	char buf[BUFSIZ], sha1buf[SHA1_DIGEST_LENGTH];
120 	size_t r, sha1buflen = 0;
121 
122 	*tot = 0;
123 	got_hash_init(&hash, GOT_HASH_SHA1);
124 
125 	for (;;) {
126 		err = cancel_cb(cancel_arg);
127 		if (err)
128 			return err;
129 
130 		r = fread(buf, 1, sizeof(buf), in);
131 		if (r == 0)
132 			break;
133 
134 		/*
135 		 * An expected SHA1 checksum sits at the end of the
136 		 * pack file.  Since we don't know the file size ahead
137 		 * of time we have to keep SHA1_DIGEST_LENGTH bytes
138 		 * buffered and avoid mixing those bytes int our hash
139 		 * computation until we know for sure that additional
140 		 * pack file data bytes follow.
141 		 *
142 		 * We can assume that BUFSIZE is greater than
143 		 * SHA1_DIGEST_LENGTH and that a short read means that
144 		 * we've reached EOF.
145 		 */
146 
147 		if (r >= sizeof(sha1buf)) {
148 			*tot += sha1buflen;
149 			got_hash_update(&hash, sha1buf, sha1buflen);
150 			if (write(outfd, sha1buf, sha1buflen) == -1)
151 				return got_error_from_errno("write");
152 
153 			r -= sizeof(sha1buf);
154 			memcpy(sha1buf, &buf[r], sizeof(sha1buf));
155 			sha1buflen = sizeof(sha1buf);
156 
157 			*tot += r;
158 			got_hash_update(&hash, buf, r);
159 			if (write(outfd, buf, r) == -1)
160 				return got_error_from_errno("write");
161 
162 			err = load_report_progress(progress_cb, progress_arg,
163 			    rl, *tot, 0, 0, 0, 0);
164 			if (err)
165 				return err;
166 
167 			continue;
168 		}
169 
170 		if (sha1buflen == 0)
171 			return got_error(GOT_ERR_BAD_PACKFILE);
172 
173 		/* short read, we've reached EOF */
174 		*tot += r;
175 		got_hash_update(&hash, sha1buf, r);
176 		if (write(outfd, sha1buf, r) == -1)
177 			return got_error_from_errno("write");
178 
179 		memmove(&sha1buf[0], &sha1buf[r], sizeof(sha1buf) - r);
180 		memcpy(&sha1buf[sizeof(sha1buf) - r], buf, r);
181 		break;
182 	}
183 
184 	if (sha1buflen == 0)
185 		return got_error(GOT_ERR_BAD_PACKFILE);
186 
187 	got_hash_final_object_id(&hash, id);
188 
189 	/* XXX SHA256 */
190 	memset(&expected_id, 0, sizeof(expected_id));
191 	memcpy(&expected_id.sha1, sha1buf, sizeof(expected_id.sha1));
192 
193 	if (got_object_id_cmp(id, &expected_id) != 0)
194 		return got_error(GOT_ERR_PACKIDX_CSUM);
195 
196 	/* re-add the expected hash at the end of the pack */
197 	if (write(outfd, sha1buf, sizeof(sha1buf)) == -1)
198 		return got_error_from_errno("write");
199 
200 	*tot += sizeof(sha1buf);
201 	err = progress_cb(progress_arg, *tot, 0, 0, 0, 0);
202 	if (err)
203 		return err;
204 
205 	return NULL;
206 }
207 
208 const struct got_error *
209 got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
210     struct got_repository *repo, int list_refs_only, int noop,
211     got_load_progress_cb progress_cb, void *progress_arg,
212     got_cancel_cb cancel_cb, void *cancel_arg)
213 {
214 	const struct got_error *err = NULL;
215 	struct got_object_id id;
216 	struct got_object *obj;
217 	struct got_packfile_hdr pack_hdr;
218 	struct got_ratelimit rl;
219 	struct imsgbuf idxibuf;
220 	const char *repo_path;
221 	char *packpath = NULL, *idxpath = NULL;
222 	char *tmppackpath = NULL, *tmpidxpath = NULL;
223 	int packfd = -1, idxfd = -1;
224 	char *spc, *refname, *id_str = NULL;
225 	char *line = NULL;
226 	size_t linesize = 0;
227 	ssize_t linelen;
228 	size_t i;
229 	ssize_t n;
230 	off_t packsiz;
231 	int tmpfds[3] = {-1, -1, -1};
232 	int imsg_idxfds[2] = {-1, -1};
233 	int ch, done, nobj, idxstatus;
234 	pid_t idxpid;
235 
236 	got_ratelimit_init(&rl, 0, 500);
237 
238 	repo_path = got_repo_get_path_git_dir(repo);
239 
240 	linelen = getline(&line, &linesize, in);
241 	if (linelen == -1) {
242 		err = got_ferror(in, GOT_ERR_IO);
243 		goto done;
244 	}
245 
246 	if (strcmp(line, GIT_BUNDLE_SIGNATURE_V2) != 0) {
247 		err = got_error(GOT_ERR_BUNDLE_FORMAT);
248 		goto done;
249 	}
250 
251 	/* Parse the prerequisite */
252 	for (;;) {
253 		ch = fgetc(in);
254 		if (ch != '-') {
255 			if (ch != EOF)
256 				ungetc(ch, in);
257 			break;
258 		}
259 
260 		linelen = getline(&line, &linesize, in);
261 		if (linelen == -1) {
262 			err = got_ferror(in, GOT_ERR_IO);
263 			goto done;
264 		}
265 
266 		if (line[linelen - 1] == '\n')
267 			line[linelen - 1] = '\0';
268 
269 		if (!got_parse_object_id(&id, line, GOT_HASH_SHA1)) {
270 			err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR);
271 			goto done;
272 		}
273 
274 		err = got_object_open(&obj, repo, &id);
275 		if (err)
276 			goto done;
277 		got_object_close(obj);
278 	}
279 
280 	/* Read references */
281 	for (;;) {
282 		struct got_object_id *id;
283 		char *dup;
284 
285 		linelen = getline(&line, &linesize, in);
286 		if (linelen == -1) {
287 			err = got_ferror(in, GOT_ERR_IO);
288 			goto done;
289 		}
290 		if (line[linelen - 1] == '\n')
291 			line[linelen - 1] = '\0';
292 		if (*line == '\0')
293 			break;
294 
295 		spc = strchr(line, ' ');
296 		if (spc == NULL) {
297 			err = got_error(GOT_ERR_IO);
298 			goto done;
299 		}
300 		*spc = '\0';
301 
302 		refname = spc + 1;
303 		if (!got_ref_name_is_valid(refname)) {
304 			err = got_error(GOT_ERR_BAD_REF_DATA);
305 			goto done;
306 		}
307 
308 		id = malloc(sizeof(*id));
309 		if (id == NULL) {
310 			err = got_error_from_errno("malloc");
311 			goto done;
312 		}
313 
314 		if (!got_parse_object_id(id, line, GOT_HASH_SHA1)) {
315 			free(id);
316 			err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
317 			goto done;
318 		}
319 
320 		dup = strdup(refname);
321 		if (dup == NULL) {
322 			free(id);
323 			err = got_error_from_errno("strdup");
324 			goto done;
325 		}
326 
327 		err = got_pathlist_append(refs_found, dup, id);
328 		if (err) {
329 			free(id);
330 			free(dup);
331 			goto done;
332 		}
333 	}
334 
335 	if (list_refs_only)
336 		goto done;
337 
338 	err = temp_file(&packfd, &tmppackpath, ".pack", repo);
339 	if (err)
340 		goto done;
341 
342 	err = temp_file(&idxfd, &tmpidxpath, ".idx", repo);
343 	if (err)
344 		goto done;
345 
346 	err = copypack(in, packfd, &packsiz, &id, &rl,
347 	    progress_cb, progress_arg, cancel_cb, cancel_arg);
348 	if (err)
349 		goto done;
350 
351 	if (lseek(packfd, 0, SEEK_SET) == -1) {
352 		err = got_error_from_errno("lseek");
353 		goto done;
354 	}
355 
356 	/* Safety checks on the pack' content. */
357 	if (packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
358 		err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
359 		goto done;
360 	}
361 
362 	n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
363 	if (n == -1) {
364 		err = got_error_from_errno("read");
365 		goto done;
366 	}
367 	if (n != ssizeof(pack_hdr)) {
368 		err = got_error(GOT_ERR_IO);
369 		goto done;
370 	}
371 	if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
372 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
373 		    "bad pack file signature");
374 		goto done;
375 	}
376 	if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
377 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
378 		    "bad pack file version");
379 		goto done;
380 	}
381 	nobj = be32toh(pack_hdr.nobjects);
382 	if (nobj == 0 &&
383 	    packsiz > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
384 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
385 		    "bad pack file with zero objects");
386 		goto done;
387 	}
388 	if (nobj != 0 &&
389 	    packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
390 		err = got_error_msg(GOT_ERR_BAD_PACKFILE,
391 		    "empty pack file with non-zero object count");
392 		goto done;
393 	}
394 
395 	/* nothing to do if there are no objects. */
396 	if (nobj == 0)
397 		goto done;
398 
399 	for (i = 0; i < nitems(tmpfds); i++) {
400 		tmpfds[i] = got_opentempfd();
401 		if (tmpfds[i] == -1) {
402 			err = got_error_from_errno("got_opentempfd");
403 			goto done;
404 		}
405 	}
406 
407 	if (lseek(packfd, 0, SEEK_SET) == -1) {
408 		err = got_error_from_errno("lseek");
409 		goto done;
410 	}
411 
412 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
413 		err = got_error_from_errno("socketpair");
414 		goto done;
415 	}
416 	idxpid = fork();
417 	if (idxpid == -1) {
418 		err= got_error_from_errno("fork");
419 		goto done;
420 	} else if (idxpid == 0)
421 		got_privsep_exec_child(imsg_idxfds,
422 		    GOT_PATH_PROG_INDEX_PACK, tmppackpath);
423 	if (close(imsg_idxfds[1]) == -1) {
424 		err = got_error_from_errno("close");
425 		goto done;
426 	}
427 	imsg_idxfds[1] = -1;
428 	imsg_init(&idxibuf, imsg_idxfds[0]);
429 
430 	err = got_privsep_send_index_pack_req(&idxibuf, id.sha1, packfd);
431 	if (err)
432 		goto done;
433 	packfd = -1;
434 
435 	err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
436 	if (err)
437 		goto done;
438 	idxfd = -1;
439 
440 	for (i = 0; i < nitems(tmpfds); i++) {
441 		err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
442 		if (err != NULL)
443 			goto done;
444 		tmpfds[i] = -1;
445 	}
446 
447 	done = 0;
448 	while (!done) {
449 		int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
450 
451 		err = got_privsep_recv_index_progress(&done, &nobj_total,
452 		    &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
453 		if (err)
454 			goto done;
455 		if (nobj_indexed != 0) {
456 			err = load_report_progress(progress_cb, progress_arg,
457 			    &rl, packsiz, nobj_total, nobj_indexed,
458 			    nobj_loose, nobj_resolved);
459 			if (err)
460 				goto done;
461 		}
462 	}
463 	if (close(imsg_idxfds[0]) == -1) {
464 		err = got_error_from_errno("close");
465 		goto done;
466 	}
467 	imsg_idxfds[0] = -1;
468 	if (waitpid(idxpid, &idxstatus, 0) == -1) {
469 		err = got_error_from_errno("waitpid");
470 		goto done;
471 	}
472 
473 	if (noop)
474 		goto done;
475 
476 	err = got_object_id_str(&id_str, &id);
477 	if (err)
478 		goto done;
479 
480 	if (asprintf(&packpath, "%s/%s/pack-%s.pack", repo_path,
481 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
482 		err = got_error_from_errno("asprintf");
483 		goto done;
484 	}
485 
486 	if (asprintf(&idxpath, "%s/%s/pack-%s.idx", repo_path,
487 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
488 		err = got_error_from_errno("asprintf");
489 		goto done;
490 	}
491 
492 	if (rename(tmppackpath, packpath) == -1) {
493 		err = got_error_from_errno3("rename", tmppackpath, packpath);
494 		goto done;
495 	}
496 	free(tmppackpath);
497 	tmppackpath = NULL;
498 
499 	if (rename(tmpidxpath, idxpath) == -1) {
500 		err = got_error_from_errno3("rename", tmpidxpath, idxpath);
501 		goto done;
502 	}
503 	free(tmpidxpath);
504 	tmpidxpath = NULL;
505 
506  done:
507 	free(line);
508 	free(packpath);
509 	free(idxpath);
510 	free(id_str);
511 
512 	if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
513 		err = got_error_from_errno2("unlink", tmppackpath);
514 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
515 		err = got_error_from_errno("close");
516 	free(tmppackpath);
517 
518 	if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
519 		err = got_error_from_errno2("unlink", tmpidxpath);
520 	if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
521 		err = got_error_from_errno("close");
522 	free(tmpidxpath);
523 
524 	if (imsg_idxfds[0] != -1 && close(imsg_idxfds[0]) == -1 && err == NULL)
525 		err = got_error_from_errno("close");
526 	if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
527 		err = got_error_from_errno("close");
528 
529 	for (i = 0; i < nitems(tmpfds); ++i)
530 		if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
531 			err = got_error_from_errno("close");
532 
533 	return err;
534 }