Blob


1 /*
2 * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/stat.h>
23 #include <stdint.h>
24 #include <errno.h>
25 #include <imsg.h>
26 #include <limits.h>
27 #include <signal.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <ctype.h>
32 #include <sha1.h>
33 #include <fcntl.h>
34 #include <unistd.h>
35 #include <zlib.h>
36 #include <err.h>
38 #include "got_error.h"
39 #include "got_object.h"
40 #include "got_path.h"
41 #include "got_version.h"
42 #include "got_fetch.h"
43 #include "got_reference.h"
45 #include "got_lib_sha1.h"
46 #include "got_lib_delta.h"
47 #include "got_lib_object.h"
48 #include "got_lib_object_parse.h"
49 #include "got_lib_privsep.h"
50 #include "got_lib_pack.h"
52 #ifndef nitems
53 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
54 #endif
56 struct got_object *indexed;
57 static int chattygot;
58 static struct got_object_id zhash = {.sha1={0}};
60 static const struct got_error *
61 readn(ssize_t *off, int fd, void *buf, size_t n)
62 {
63 ssize_t r;
65 *off = 0;
66 while (*off != n) {
67 r = read(fd, buf + *off, n - *off);
68 if (r == -1)
69 return got_error_from_errno("read");
70 if (r == 0)
71 return NULL;
72 *off += r;
73 }
74 return NULL;
75 }
77 static const struct got_error *
78 flushpkt(int fd)
79 {
80 ssize_t w;
82 if (chattygot > 1)
83 fprintf(stderr, "%s: writepkt: 0000\n", getprogname());
85 w = write(fd, "0000", 4);
86 if (w == -1)
87 return got_error_from_errno("write");
88 if (w != 4)
89 return got_error(GOT_ERR_IO);
90 return NULL;
91 }
93 /*
94 * Packet header contains a 4-byte hexstring which specifies the length
95 * of data which follows.
96 */
97 static const struct got_error *
98 read_pkthdr(int *datalen, int fd)
99 {
100 static const struct got_error *err = NULL;
101 char lenstr[5];
102 long len;
103 char *e;
104 int n, i;
105 ssize_t r;
107 *datalen = 0;
109 err = readn(&r, fd, lenstr, 4);
110 if (err)
111 return err;
112 if (r == 0) {
113 /* implicit "0000" */
114 if (chattygot > 1)
115 fprintf(stderr, "%s: readpkt: 0000\n", getprogname());
116 return NULL;
118 if (r != 4)
119 return got_error_msg(GOT_ERR_BAD_PACKET,
120 "wrong packet header length");
122 lenstr[4] = '\0';
123 for (i = 0; i < 4; i++) {
124 if (!isprint((unsigned char)lenstr[i]))
125 return got_error_msg(GOT_ERR_BAD_PACKET,
126 "unprintable character in packet length field");
128 for (i = 0; i < 4; i++) {
129 if (!isxdigit((unsigned char)lenstr[i])) {
130 if (chattygot)
131 fprintf(stderr, "%s: bad length: '%s'\n",
132 getprogname(), lenstr);
133 return got_error_msg(GOT_ERR_BAD_PACKET,
134 "packet length not specified in hex");
137 errno = 0;
138 len = strtol(lenstr, &e, 16);
139 if (lenstr[0] == '\0' || *e != '\0')
140 return got_error(GOT_ERR_BAD_PACKET);
141 if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN))
142 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
143 if (len > INT_MAX || len < INT_MIN)
144 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
145 n = len;
146 if (n == 0)
147 return NULL;
148 if (n <= 4)
149 return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short");
150 n -= 4;
152 *datalen = n;
153 return NULL;
156 static const struct got_error *
157 readpkt(int *outlen, int fd, char *buf, int buflen)
159 const struct got_error *err = NULL;
160 int datalen, i;
161 ssize_t n;
163 err = read_pkthdr(&datalen, fd);
164 if (err)
165 return err;
167 if (datalen > buflen)
168 return got_error(GOT_ERR_NO_SPACE);
170 err = readn(&n, fd, buf, datalen);
171 if (err)
172 return err;
173 if (n != datalen)
174 return got_error_msg(GOT_ERR_BAD_PACKET, "short packet");
176 if (chattygot > 1) {
177 fprintf(stderr, "%s: readpkt: %zd:\t", getprogname(), n);
178 for (i = 0; i < n; i++) {
179 if (isprint(buf[i]))
180 fputc(buf[i], stderr);
181 else
182 fprintf(stderr, "[0x%.2x]", buf[i]);
184 fputc('\n', stderr);
187 *outlen = n;
188 return NULL;
191 static const struct got_error *
192 writepkt(int fd, char *buf, int nbuf)
194 char len[5];
195 int i;
196 ssize_t w;
198 if (snprintf(len, sizeof(len), "%04x", nbuf + 4) >= sizeof(len))
199 return got_error(GOT_ERR_NO_SPACE);
200 w = write(fd, len, 4);
201 if (w == -1)
202 return got_error_from_errno("write");
203 if (w != 4)
204 return got_error(GOT_ERR_IO);
205 w = write(fd, buf, nbuf);
206 if (w == -1)
207 return got_error_from_errno("write");
208 if (w != nbuf)
209 return got_error(GOT_ERR_IO);
210 if (chattygot > 1) {
211 fprintf(stderr, "%s: writepkt: %s:\t", getprogname(), len);
212 for (i = 0; i < nbuf; i++) {
213 if (isprint(buf[i]))
214 fputc(buf[i], stderr);
215 else
216 fprintf(stderr, "[0x%.2x]", buf[i]);
218 fputc('\n', stderr);
220 return NULL;
223 static void
224 match_remote_ref(struct got_pathlist_head *have_refs,
225 struct got_object_id *my_id, char *refname)
227 struct got_pathlist_entry *pe;
229 /* XXX zero-hash signifies we don't have this ref;
230 * we should use a flag instead */
231 memset(my_id, 0, sizeof(*my_id));
233 TAILQ_FOREACH(pe, have_refs, entry) {
234 struct got_object_id *id = pe->data;
235 if (strcmp(pe->path, refname) == 0) {
236 memcpy(my_id, id, sizeof(*my_id));
237 break;
242 static int
243 match_branch(const char *branch, const char *wanted_branch)
245 if (strncmp(branch, "refs/heads/", 11) != 0)
246 return 0;
248 if (strncmp(wanted_branch, "refs/heads/", 11) == 0)
249 wanted_branch += 11;
251 return (strcmp(branch + 11, wanted_branch) == 0);
254 static int
255 match_wanted_ref(const char *refname, const char *wanted_ref)
257 if (strncmp(refname, "refs/", 5) != 0)
258 return 0;
259 refname += 5;
261 /*
262 * Prevent fetching of references that won't make any
263 * sense outside of the remote repository's context.
264 */
265 if (strncmp(refname, "got/", 4) == 0)
266 return 0;
267 if (strncmp(refname, "remotes/", 8) == 0)
268 return 0;
270 if (strncmp(wanted_ref, "refs/", 5) == 0)
271 wanted_ref += 5;
273 /* Allow prefix match. */
274 if (got_path_is_child(refname, wanted_ref, strlen(wanted_ref)))
275 return 1;
277 /* Allow exact match. */
278 return (strcmp(refname, wanted_ref) == 0);
281 static const struct got_error *
282 tokenize_refline(char **tokens, char *line, int len, int maxtokens)
284 const struct got_error *err = NULL;
285 char *p;
286 size_t i, n = 0;
288 for (i = 0; i < maxtokens; i++)
289 tokens[i] = NULL;
291 for (i = 0; n < len && i < maxtokens; i++) {
292 while (isspace(*line)) {
293 line++;
294 n++;
296 p = line;
297 while (*line != '\0' && n < len &&
298 (!isspace(*line) || i == maxtokens - 1)) {
299 line++;
300 n++;
302 tokens[i] = strndup(p, line - p);
303 if (tokens[i] == NULL) {
304 err = got_error_from_errno("strndup");
305 goto done;
307 /* Skip \0 field-delimiter at end of token. */
308 while (line[0] == '\0' && n < len) {
309 line++;
310 n++;
313 if (i <= 2)
314 err = got_error(GOT_ERR_BAD_PACKET);
315 done:
316 if (err) {
317 int j;
318 for (j = 0; j < i; j++) {
319 free(tokens[j]);
320 tokens[j] = NULL;
323 return err;
326 static const struct got_error *
327 parse_refline(char **id_str, char **refname, char **server_capabilities,
328 char *line, int len)
330 const struct got_error *err = NULL;
331 char *tokens[3];
333 err = tokenize_refline(tokens, line, len, nitems(tokens));
334 if (err)
335 return err;
337 if (tokens[0])
338 *id_str = tokens[0];
339 if (tokens[1])
340 *refname = tokens[1];
341 if (tokens[2]) {
342 char *p;
343 *server_capabilities = tokens[2];
344 p = strrchr(*server_capabilities, '\n');
345 if (p)
346 *p = '\0';
349 return NULL;
352 #define GOT_CAPA_AGENT "agent"
353 #define GOT_CAPA_OFS_DELTA "ofs-delta"
354 #define GOT_CAPA_SIDE_BAND_64K "side-band-64k"
356 #define GOT_SIDEBAND_PACKFILE_DATA 1
357 #define GOT_SIDEBAND_PROGRESS_INFO 2
358 #define GOT_SIDEBAND_ERROR_INFO 3
361 struct got_capability {
362 const char *key;
363 const char *value;
364 };
365 static const struct got_capability got_capabilities[] = {
366 { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
367 { GOT_CAPA_OFS_DELTA, NULL },
368 { GOT_CAPA_SIDE_BAND_64K, NULL },
369 };
371 static const struct got_error *
372 match_capability(char **my_capabilities, const char *capa,
373 const struct got_capability *mycapa)
375 char *equalsign;
376 char *s;
378 equalsign = strchr(capa, '=');
379 if (equalsign) {
380 if (strncmp(capa, mycapa->key, equalsign - capa) != 0)
381 return NULL;
382 } else {
383 if (strcmp(capa, mycapa->key) != 0)
384 return NULL;
387 if (asprintf(&s, "%s %s%s%s",
388 *my_capabilities != NULL ? *my_capabilities : "",
389 mycapa->key,
390 mycapa->value != NULL ? "=" : "",
391 mycapa->value != NULL? mycapa->value : "") == -1)
392 return got_error_from_errno("asprintf");
394 free(*my_capabilities);
395 *my_capabilities = s;
396 return NULL;
399 static const struct got_error *
400 add_symref(struct got_pathlist_head *symrefs, char *capa)
402 const struct got_error *err = NULL;
403 char *colon, *name = NULL, *target = NULL;
405 /* Need at least "A:B" */
406 if (strlen(capa) < 3)
407 return NULL;
409 colon = strchr(capa, ':');
410 if (colon == NULL)
411 return NULL;
413 *colon = '\0';
414 name = strdup(capa);
415 if (name == NULL)
416 return got_error_from_errno("strdup");
418 target = strdup(colon + 1);
419 if (target == NULL) {
420 err = got_error_from_errno("strdup");
421 goto done;
424 /* We can't validate the ref itself here. The main process will. */
425 err = got_pathlist_append(symrefs, name, target);
426 done:
427 if (err) {
428 free(name);
429 free(target);
431 return err;
434 static const struct got_error *
435 match_capabilities(char **my_capabilities, struct got_pathlist_head *symrefs,
436 char *server_capabilities)
438 const struct got_error *err = NULL;
439 char *capa, *equalsign;
440 size_t i;
442 *my_capabilities = NULL;
443 do {
444 capa = strsep(&server_capabilities, " ");
445 if (capa == NULL)
446 return NULL;
448 equalsign = strchr(capa, '=');
449 if (equalsign != NULL &&
450 strncmp(capa, "symref", equalsign - capa) == 0) {
451 err = add_symref(symrefs, equalsign + 1);
452 if (err)
453 break;
454 continue;
457 for (i = 0; i < nitems(got_capabilities); i++) {
458 err = match_capability(my_capabilities,
459 capa, &got_capabilities[i]);
460 if (err)
461 break;
463 } while (capa);
465 if (*my_capabilities == NULL) {
466 *my_capabilities = strdup("");
467 if (*my_capabilities == NULL)
468 err = got_error_from_errno("strdup");
470 return err;
473 static const struct got_error *
474 send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, size_t msglen)
476 if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
477 return got_error(GOT_ERR_NO_SPACE);
479 if (msglen == 0)
480 return NULL;
482 if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
483 msg, msglen) == -1)
484 return got_error_from_errno(
485 "imsg_compose FETCH_SERVER_PROGRESS");
487 return got_privsep_flush_imsg(ibuf);
490 static const struct got_error *
491 send_fetch_download_progress(struct imsgbuf *ibuf, off_t bytes)
493 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DOWNLOAD_PROGRESS, 0, 0, -1,
494 &bytes, sizeof(bytes)) == -1)
495 return got_error_from_errno(
496 "imsg_compose FETCH_DOWNLOAD_PROGRESS");
498 return got_privsep_flush_imsg(ibuf);
501 static const struct got_error *
502 send_fetch_done(struct imsgbuf *ibuf, uint8_t *pack_sha1)
504 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
505 pack_sha1, SHA1_DIGEST_LENGTH) == -1)
506 return got_error_from_errno("imsg_compose FETCH");
507 return got_privsep_flush_imsg(ibuf);
512 static const struct got_error *
513 fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
515 size_t i;
517 if (len == 0)
518 return NULL;
520 /*
521 * Truncate messages which exceed the maximum imsg payload size.
522 * Server may send up to 64k.
523 */
524 if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
525 len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
527 /* Only allow printable ASCII. */
528 for (i = 0; i < len; i++) {
529 if (isprint((unsigned char)buf[i]) ||
530 isspace((unsigned char)buf[i]))
531 continue;
532 return got_error_msg(GOT_ERR_BAD_PACKET,
533 "non-printable progress message received from server");
536 return send_fetch_server_progress(ibuf, buf, len);
539 static const struct got_error *
540 fetch_error(const char *buf, size_t len)
542 static char msg[1024];
543 size_t i;
545 for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
546 if (!isprint(buf[i]))
547 return got_error_msg(GOT_ERR_BAD_PACKET,
548 "non-printable error message received from server");
549 msg[i] = buf[i];
551 msg[i] = '\0';
552 return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
555 static const struct got_error *
556 send_fetch_symrefs(struct imsgbuf *ibuf, struct got_pathlist_head *symrefs)
558 const struct got_error *err = NULL;
559 struct ibuf *wbuf;
560 size_t len, nsymrefs = 0;
561 struct got_pathlist_entry *pe;
563 len = sizeof(struct got_imsg_fetch_symrefs);
564 TAILQ_FOREACH(pe, symrefs, entry) {
565 const char *target = pe->data;
566 len += sizeof(struct got_imsg_fetch_symref) +
567 pe->path_len + strlen(target);
568 nsymrefs++;
571 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
572 return got_error(GOT_ERR_NO_SPACE);
574 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_SYMREFS, 0, 0, len);
575 if (wbuf == NULL)
576 return got_error_from_errno("imsg_create FETCH_SYMREFS");
578 /* Keep in sync with struct got_imsg_fetch_symrefs definition! */
579 if (imsg_add(wbuf, &nsymrefs, sizeof(nsymrefs)) == -1) {
580 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
581 ibuf_free(wbuf);
582 return err;
585 TAILQ_FOREACH(pe, symrefs, entry) {
586 const char *name = pe->path;
587 size_t name_len = pe->path_len;
588 const char *target = pe->data;
589 size_t target_len = strlen(target);
591 /* Keep in sync with struct got_imsg_fetch_symref definition! */
592 if (imsg_add(wbuf, &name_len, sizeof(name_len)) == -1) {
593 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
594 ibuf_free(wbuf);
595 return err;
597 if (imsg_add(wbuf, &target_len, sizeof(target_len)) == -1) {
598 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
599 ibuf_free(wbuf);
600 return err;
602 if (imsg_add(wbuf, name, name_len) == -1) {
603 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
604 ibuf_free(wbuf);
605 return err;
607 if (imsg_add(wbuf, target, target_len) == -1) {
608 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
609 ibuf_free(wbuf);
610 return err;
614 wbuf->fd = -1;
615 imsg_close(ibuf, wbuf);
616 return got_privsep_flush_imsg(ibuf);
619 static const struct got_error *
620 send_fetch_ref(struct imsgbuf *ibuf, struct got_object_id *refid,
621 const char *refname)
623 const struct got_error *err = NULL;
624 struct ibuf *wbuf;
625 size_t len, reflen = strlen(refname);
627 len = sizeof(struct got_imsg_fetch_ref) + reflen;
628 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
629 return got_error(GOT_ERR_NO_SPACE);
631 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_REF, 0, 0, len);
632 if (wbuf == NULL)
633 return got_error_from_errno("imsg_create FETCH_REF");
635 /* Keep in sync with struct got_imsg_fetch_ref definition! */
636 if (imsg_add(wbuf, refid->sha1, SHA1_DIGEST_LENGTH) == -1) {
637 err = got_error_from_errno("imsg_add FETCH_REF");
638 ibuf_free(wbuf);
639 return err;
641 if (imsg_add(wbuf, refname, reflen) == -1) {
642 err = got_error_from_errno("imsg_add FETCH_REF");
643 ibuf_free(wbuf);
644 return err;
647 wbuf->fd = -1;
648 imsg_close(ibuf, wbuf);
649 return got_privsep_flush_imsg(ibuf);
652 static const struct got_error *
653 fetch_pack(int fd, int packfd, uint8_t *pack_sha1,
654 struct got_pathlist_head *have_refs, int fetch_all_branches,
655 struct got_pathlist_head *wanted_branches,
656 struct got_pathlist_head *wanted_refs, int list_refs_only,
657 struct imsgbuf *ibuf)
659 const struct got_error *err = NULL;
660 char buf[GOT_FETCH_PKTMAX];
661 char hashstr[SHA1_DIGEST_STRING_LENGTH];
662 struct got_object_id *have, *want;
663 int is_firstpkt = 1, nref = 0, refsz = 16;
664 int i, n, nwant = 0, nhave = 0, acked = 0;
665 off_t packsz = 0, last_reported_packsz = 0;
666 char *id_str = NULL, *refname = NULL;
667 char *server_capabilities = NULL, *my_capabilities = NULL;
668 const char *default_branch = NULL;
669 struct got_pathlist_head symrefs;
670 struct got_pathlist_entry *pe;
671 int sent_my_capabilites = 0, have_sidebands = 0;
672 int found_branch = 0;
673 SHA1_CTX sha1_ctx;
674 uint8_t sha1_buf[SHA1_DIGEST_LENGTH];
675 size_t sha1_buf_len = 0;
676 ssize_t w;
678 TAILQ_INIT(&symrefs);
679 SHA1Init(&sha1_ctx);
681 have = malloc(refsz * sizeof(have[0]));
682 if (have == NULL)
683 return got_error_from_errno("malloc");
684 want = malloc(refsz * sizeof(want[0]));
685 if (want == NULL) {
686 err = got_error_from_errno("malloc");
687 goto done;
689 while (1) {
690 err = readpkt(&n, fd, buf, sizeof(buf));
691 if (err)
692 goto done;
693 if (n == 0)
694 break;
695 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
696 err = fetch_error(&buf[4], n - 4);
697 goto done;
699 err = parse_refline(&id_str, &refname, &server_capabilities,
700 buf, n);
701 if (err)
702 goto done;
703 if (is_firstpkt) {
704 if (chattygot && server_capabilities[0] != '\0')
705 fprintf(stderr, "%s: server capabilities: %s\n",
706 getprogname(), server_capabilities);
707 err = match_capabilities(&my_capabilities, &symrefs,
708 server_capabilities);
709 if (err)
710 goto done;
711 if (chattygot)
712 fprintf(stderr, "%s: my capabilities:%s\n",
713 getprogname(), my_capabilities != NULL ?
714 my_capabilities : "");
715 err = send_fetch_symrefs(ibuf, &symrefs);
716 if (err)
717 goto done;
718 is_firstpkt = 0;
719 if (!fetch_all_branches) {
720 TAILQ_FOREACH(pe, &symrefs, entry) {
721 const char *name = pe->path;
722 const char *symref_target = pe->data;
723 if (strcmp(name, GOT_REF_HEAD) != 0)
724 continue;
725 default_branch = symref_target;
726 break;
729 continue;
731 if (strstr(refname, "^{}")) {
732 if (chattygot) {
733 fprintf(stderr, "%s: ignoring %s\n",
734 getprogname(), refname);
736 continue;
739 if (strncmp(refname, "refs/heads/", 11) == 0) {
740 if (fetch_all_branches || list_refs_only) {
741 found_branch = 1;
742 } else if (!TAILQ_EMPTY(wanted_branches)) {
743 TAILQ_FOREACH(pe, wanted_branches, entry) {
744 if (match_branch(refname, pe->path))
745 break;
747 if (pe == NULL) {
748 if (chattygot) {
749 fprintf(stderr,
750 "%s: ignoring %s\n",
751 getprogname(), refname);
753 continue;
755 found_branch = 1;
756 } else if (default_branch != NULL) {
757 if (!match_branch(refname, default_branch)) {
758 if (chattygot) {
759 fprintf(stderr,
760 "%s: ignoring %s\n",
761 getprogname(), refname);
763 continue;
765 found_branch = 1;
767 } else if (strncmp(refname, "refs/tags/", 10) != 0) {
768 if (!TAILQ_EMPTY(wanted_refs)) {
769 TAILQ_FOREACH(pe, wanted_refs, entry) {
770 if (match_wanted_ref(refname, pe->path))
771 break;
773 if (pe == NULL) {
774 if (chattygot) {
775 fprintf(stderr,
776 "%s: ignoring %s\n",
777 getprogname(), refname);
779 continue;
781 found_branch = 1;
782 } else if (!list_refs_only) {
783 if (chattygot) {
784 fprintf(stderr, "%s: ignoring %s\n",
785 getprogname(), refname);
787 continue;
791 if (refsz == nref + 1) {
792 refsz *= 2;
793 have = reallocarray(have, refsz, sizeof(have[0]));
794 if (have == NULL) {
795 err = got_error_from_errno("reallocarray");
796 goto done;
798 want = reallocarray(want, refsz, sizeof(want[0]));
799 if (want == NULL) {
800 err = got_error_from_errno("reallocarray");
801 goto done;
804 if (!got_parse_sha1_digest(want[nref].sha1, id_str)) {
805 err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
806 goto done;
808 match_remote_ref(have_refs, &have[nref], refname);
809 err = send_fetch_ref(ibuf, &want[nref], refname);
810 if (err)
811 goto done;
813 if (chattygot)
814 fprintf(stderr, "%s: %s will be fetched\n",
815 getprogname(), refname);
816 if (chattygot > 1) {
817 char *theirs, *mine;
818 err = got_object_id_str(&theirs, &want[nref]);
819 if (err)
820 goto done;
821 err = got_object_id_str(&mine, &have[nref]);
822 if (err) {
823 free(theirs);
824 goto done;
826 fprintf(stderr, "%s: remote: %s\n%s: local: %s\n",
827 getprogname(), theirs, getprogname(), mine);
828 free(theirs);
829 free(mine);
831 nref++;
834 if (list_refs_only)
835 goto done;
837 /* Abort if we haven't found any branch to fetch. */
838 if (!found_branch) {
839 err = got_error(GOT_ERR_FETCH_NO_BRANCH);
840 goto done;
843 for (i = 0; i < nref; i++) {
844 if (got_object_id_cmp(&have[i], &want[i]) == 0)
845 continue;
846 got_sha1_digest_to_str(want[i].sha1, hashstr, sizeof(hashstr));
847 n = snprintf(buf, sizeof(buf), "want %s%s\n", hashstr,
848 sent_my_capabilites || my_capabilities == NULL ?
849 "" : my_capabilities);
850 if (n >= sizeof(buf)) {
851 err = got_error(GOT_ERR_NO_SPACE);
852 goto done;
854 err = writepkt(fd, buf, n);
855 if (err)
856 goto done;
857 sent_my_capabilites = 1;
858 nwant++;
860 err = flushpkt(fd);
861 if (err)
862 goto done;
864 if (nwant == 0)
865 goto done;
867 for (i = 0; i < nref; i++) {
868 if (got_object_id_cmp(&have[i], &zhash) == 0)
869 continue;
870 got_sha1_digest_to_str(have[i].sha1, hashstr, sizeof(hashstr));
871 n = snprintf(buf, sizeof(buf), "have %s\n", hashstr);
872 if (n >= sizeof(buf)) {
873 err = got_error(GOT_ERR_NO_SPACE);
874 goto done;
876 err = writepkt(fd, buf, n);
877 if (err)
878 goto done;
879 nhave++;
882 while (nhave > 0 && !acked) {
883 struct got_object_id common_id;
885 /* The server should ACK the object IDs we need. */
886 err = readpkt(&n, fd, buf, sizeof(buf));
887 if (err)
888 goto done;
889 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
890 err = fetch_error(&buf[4], n - 4);
891 goto done;
893 if (n >= 4 && strncmp(buf, "NAK\n", 4) == 0) {
894 /* Server has not located our objects yet. */
895 continue;
897 if (n < 4 + SHA1_DIGEST_STRING_LENGTH ||
898 strncmp(buf, "ACK ", 4) != 0) {
899 err = got_error_msg(GOT_ERR_BAD_PACKET,
900 "unexpected message from server");
901 goto done;
903 if (!got_parse_sha1_digest(common_id.sha1, buf + 4)) {
904 err = got_error_msg(GOT_ERR_BAD_PACKET,
905 "bad object ID in ACK packet from server");
906 goto done;
908 acked++;
911 n = snprintf(buf, sizeof(buf), "done\n");
912 err = writepkt(fd, buf, n);
913 if (err)
914 goto done;
916 if (nhave == 0) {
917 err = readpkt(&n, fd, buf, sizeof(buf));
918 if (err)
919 goto done;
920 if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
921 err = got_error_msg(GOT_ERR_BAD_PACKET,
922 "unexpected message from server");
923 goto done;
927 if (chattygot)
928 fprintf(stderr, "%s: fetching...\n", getprogname());
930 if (my_capabilities != NULL &&
931 strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
932 have_sidebands = 1;
934 while (1) {
935 ssize_t r = 0;
936 int datalen = -1;
938 if (have_sidebands) {
939 err = read_pkthdr(&datalen, fd);
940 if (err)
941 goto done;
942 if (datalen <= 0)
943 break;
945 /* Read sideband channel ID (one byte). */
946 r = read(fd, buf, 1);
947 if (r == -1) {
948 err = got_error_from_errno("read");
949 goto done;
951 if (r != 1) {
952 err = got_error_msg(GOT_ERR_BAD_PACKET,
953 "short packet");
954 goto done;
956 if (datalen > sizeof(buf) - 5) {
957 err = got_error_msg(GOT_ERR_BAD_PACKET,
958 "bad packet length");
959 goto done;
961 datalen--; /* sideband ID has been read */
962 if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
963 /* Read packfile data. */
964 err = readn(&r, fd, buf, datalen);
965 if (err)
966 goto done;
967 if (r != datalen) {
968 err = got_error_msg(GOT_ERR_BAD_PACKET,
969 "packet too short");
970 goto done;
972 } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
973 err = readn(&r, fd, buf, datalen);
974 if (err)
975 goto done;
976 if (r != datalen) {
977 err = got_error_msg(GOT_ERR_BAD_PACKET,
978 "packet too short");
979 goto done;
981 err = fetch_progress(ibuf, buf, r);
982 if (err)
983 goto done;
984 continue;
985 } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
986 err = readn(&r, fd, buf, datalen);
987 if (err)
988 goto done;
989 if (r != datalen) {
990 err = got_error_msg(GOT_ERR_BAD_PACKET,
991 "packet too short");
992 goto done;
994 err = fetch_error(buf, r);
995 goto done;
996 } else if (buf[0] == 'A') {
997 err = readn(&r, fd, buf, datalen);
998 if (err)
999 goto done;
1000 if (r != datalen) {
1001 err = got_error_msg(GOT_ERR_BAD_PACKET,
1002 "packet too short");
1003 goto done;
1006 * Git server responds with ACK after 'done'
1007 * even though multi_ack is disabled?!?
1009 buf[r] = '\0';
1010 if (strncmp(buf, "CK ", 3) == 0)
1011 continue; /* ignore */
1012 err = got_error_msg(GOT_ERR_BAD_PACKET,
1013 "unexpected message from server");
1014 goto done;
1015 } else {
1016 err = got_error_msg(GOT_ERR_BAD_PACKET,
1017 "unknown side-band received from server");
1018 goto done;
1020 } else {
1021 /* No sideband channel. Every byte is packfile data. */
1022 err = readn(&r, fd, buf, sizeof buf);
1023 if (err)
1024 goto done;
1025 if (r <= 0)
1026 break;
1030 * An expected SHA1 checksum sits at the end of the pack file.
1031 * Since we don't know the file size ahead of time we have to
1032 * keep SHA1_DIGEST_LENGTH bytes buffered and avoid mixing
1033 * those bytes into our SHA1 checksum computation until we
1034 * know for sure that additional pack file data bytes follow.
1036 * We can assume r > 0 since otherwise the loop would exit.
1038 if (r < SHA1_DIGEST_LENGTH) {
1039 if (sha1_buf_len < SHA1_DIGEST_LENGTH) {
1041 * If there's enough buffered + read data to
1042 * fill up the buffer then shift a sufficient
1043 * amount of bytes out at the front to make
1044 * room, mixing those bytes into the checksum.
1046 while (sha1_buf_len > 0 &&
1047 sha1_buf_len + r > SHA1_DIGEST_LENGTH) {
1048 SHA1Update(&sha1_ctx, sha1_buf, 1);
1049 memmove(sha1_buf, sha1_buf + 1, 1);
1050 sha1_buf_len--;
1053 /* Buffer potential checksum bytes. */
1054 memcpy(sha1_buf + sha1_buf_len, buf, r);
1055 sha1_buf_len += r;
1056 } else {
1058 * Mix in previously buffered bytes which
1059 * are not part of the checksum after all.
1061 SHA1Update(&sha1_ctx, sha1_buf, r);
1063 /* Update potential checksum buffer. */
1064 memmove(sha1_buf, sha1_buf + r,
1065 sha1_buf_len - r);
1066 memcpy(sha1_buf + sha1_buf_len - r, buf, r);
1068 } else {
1069 /* Mix in any previously buffered bytes. */
1070 SHA1Update(&sha1_ctx, sha1_buf, sha1_buf_len);
1072 /* Mix in bytes read minus potential checksum bytes. */
1073 SHA1Update(&sha1_ctx, buf, r - SHA1_DIGEST_LENGTH);
1075 /* Buffer potential checksum bytes. */
1076 memcpy(sha1_buf, buf + r - SHA1_DIGEST_LENGTH,
1077 SHA1_DIGEST_LENGTH);
1078 sha1_buf_len = SHA1_DIGEST_LENGTH;
1081 /* Write packfile data to temporary pack file. */
1082 w = write(packfd, buf, r);
1083 if (w == -1) {
1084 err = got_error_from_errno("write");
1085 goto done;
1087 if (w != r) {
1088 err = got_error(GOT_ERR_IO);
1089 goto done;
1091 packsz += w;
1093 /* Don't send too many progress privsep messages. */
1094 if (packsz > last_reported_packsz + 1024) {
1095 err = send_fetch_download_progress(ibuf, packsz);
1096 if (err)
1097 goto done;
1098 last_reported_packsz = packsz;
1101 err = send_fetch_download_progress(ibuf, packsz);
1102 if (err)
1103 goto done;
1105 SHA1Final(pack_sha1, &sha1_ctx);
1106 if (sha1_buf_len != SHA1_DIGEST_LENGTH ||
1107 memcmp(pack_sha1, sha1_buf, sha1_buf_len) != 0) {
1108 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
1109 "pack file checksum mismatch");
1111 done:
1112 TAILQ_FOREACH(pe, &symrefs, entry) {
1113 free((void *)pe->path);
1114 free(pe->data);
1116 got_pathlist_free(&symrefs);
1117 free(have);
1118 free(want);
1119 free(id_str);
1120 free(refname);
1121 free(server_capabilities);
1122 return err;
1126 int
1127 main(int argc, char **argv)
1129 const struct got_error *err = NULL;
1130 int fetchfd, packfd = -1, i;
1131 uint8_t pack_sha1[SHA1_DIGEST_LENGTH];
1132 struct imsgbuf ibuf;
1133 struct imsg imsg;
1134 struct got_pathlist_head have_refs;
1135 struct got_pathlist_head wanted_branches;
1136 struct got_pathlist_head wanted_refs;
1137 struct got_pathlist_entry *pe;
1138 struct got_imsg_fetch_request fetch_req;
1139 struct got_imsg_fetch_have_ref href;
1140 struct got_imsg_fetch_wanted_branch wbranch;
1141 struct got_imsg_fetch_wanted_ref wref;
1142 size_t datalen;
1143 #if 0
1144 static int attached;
1145 while (!attached)
1146 sleep (1);
1147 #endif
1149 TAILQ_INIT(&have_refs);
1150 TAILQ_INIT(&wanted_branches);
1151 TAILQ_INIT(&wanted_refs);
1153 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1154 #ifndef PROFILE
1155 /* revoke access to most system calls */
1156 if (pledge("stdio recvfd", NULL) == -1) {
1157 err = got_error_from_errno("pledge");
1158 got_privsep_send_error(&ibuf, err);
1159 return 1;
1161 #endif
1162 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1163 if (err) {
1164 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1165 err = NULL;
1166 goto done;
1168 if (imsg.hdr.type == GOT_IMSG_STOP)
1169 goto done;
1170 if (imsg.hdr.type != GOT_IMSG_FETCH_REQUEST) {
1171 err = got_error(GOT_ERR_PRIVSEP_MSG);
1172 goto done;
1174 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1175 if (datalen < sizeof(fetch_req)) {
1176 err = got_error(GOT_ERR_PRIVSEP_LEN);
1177 goto done;
1179 memcpy(&fetch_req, imsg.data, sizeof(fetch_req));
1180 fetchfd = imsg.fd;
1181 imsg_free(&imsg);
1183 if (fetch_req.verbosity > 0)
1184 chattygot += fetch_req.verbosity;
1186 for (i = 0; i < fetch_req.n_have_refs; i++) {
1187 struct got_object_id *id;
1188 char *refname;
1190 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1191 if (err) {
1192 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1193 err = NULL;
1194 goto done;
1196 if (imsg.hdr.type == GOT_IMSG_STOP)
1197 goto done;
1198 if (imsg.hdr.type != GOT_IMSG_FETCH_HAVE_REF) {
1199 err = got_error(GOT_ERR_PRIVSEP_MSG);
1200 goto done;
1202 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1203 if (datalen < sizeof(href)) {
1204 err = got_error(GOT_ERR_PRIVSEP_LEN);
1205 goto done;
1207 memcpy(&href, imsg.data, sizeof(href));
1208 if (datalen - sizeof(href) < href.name_len) {
1209 err = got_error(GOT_ERR_PRIVSEP_LEN);
1210 goto done;
1212 refname = malloc(href.name_len + 1);
1213 if (refname == NULL) {
1214 err = got_error_from_errno("malloc");
1215 goto done;
1217 memcpy(refname, imsg.data + sizeof(href), href.name_len);
1218 refname[href.name_len] = '\0';
1220 id = malloc(sizeof(*id));
1221 if (id == NULL) {
1222 free(refname);
1223 err = got_error_from_errno("malloc");
1224 goto done;
1226 memcpy(id->sha1, href.id, SHA1_DIGEST_LENGTH);
1227 err = got_pathlist_append(&have_refs, refname, id);
1228 if (err) {
1229 free(refname);
1230 free(id);
1231 goto done;
1234 imsg_free(&imsg);
1237 for (i = 0; i < fetch_req.n_wanted_branches; i++) {
1238 char *refname;
1240 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1241 if (err) {
1242 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1243 err = NULL;
1244 goto done;
1246 if (imsg.hdr.type == GOT_IMSG_STOP)
1247 goto done;
1248 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_BRANCH) {
1249 err = got_error(GOT_ERR_PRIVSEP_MSG);
1250 goto done;
1252 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1253 if (datalen < sizeof(wbranch)) {
1254 err = got_error(GOT_ERR_PRIVSEP_LEN);
1255 goto done;
1257 memcpy(&wbranch, imsg.data, sizeof(wbranch));
1258 if (datalen - sizeof(wbranch) < wbranch.name_len) {
1259 err = got_error(GOT_ERR_PRIVSEP_LEN);
1260 goto done;
1262 refname = malloc(wbranch.name_len + 1);
1263 if (refname == NULL) {
1264 err = got_error_from_errno("malloc");
1265 goto done;
1267 memcpy(refname, imsg.data + sizeof(wbranch), wbranch.name_len);
1268 refname[wbranch.name_len] = '\0';
1270 err = got_pathlist_append(&wanted_branches, refname, NULL);
1271 if (err) {
1272 free(refname);
1273 goto done;
1276 imsg_free(&imsg);
1279 for (i = 0; i < fetch_req.n_wanted_refs; i++) {
1280 char *refname;
1282 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1283 if (err) {
1284 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1285 err = NULL;
1286 goto done;
1288 if (imsg.hdr.type == GOT_IMSG_STOP)
1289 goto done;
1290 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_REF) {
1291 err = got_error(GOT_ERR_PRIVSEP_MSG);
1292 goto done;
1294 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1295 if (datalen < sizeof(wref)) {
1296 err = got_error(GOT_ERR_PRIVSEP_LEN);
1297 goto done;
1299 memcpy(&wref, imsg.data, sizeof(wref));
1300 if (datalen - sizeof(wref) < wref.name_len) {
1301 err = got_error(GOT_ERR_PRIVSEP_LEN);
1302 goto done;
1304 refname = malloc(wref.name_len + 1);
1305 if (refname == NULL) {
1306 err = got_error_from_errno("malloc");
1307 goto done;
1309 memcpy(refname, imsg.data + sizeof(wref), wref.name_len);
1310 refname[wref.name_len] = '\0';
1312 err = got_pathlist_append(&wanted_refs, refname, NULL);
1313 if (err) {
1314 free(refname);
1315 goto done;
1318 imsg_free(&imsg);
1321 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1322 if (err) {
1323 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1324 err = NULL;
1325 goto done;
1327 if (imsg.hdr.type == GOT_IMSG_STOP)
1328 goto done;
1329 if (imsg.hdr.type != GOT_IMSG_FETCH_OUTFD) {
1330 err = got_error(GOT_ERR_PRIVSEP_MSG);
1331 goto done;
1333 if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1334 err = got_error(GOT_ERR_PRIVSEP_LEN);
1335 goto done;
1337 packfd = imsg.fd;
1339 err = fetch_pack(fetchfd, packfd, pack_sha1, &have_refs,
1340 fetch_req.fetch_all_branches, &wanted_branches,
1341 &wanted_refs, fetch_req.list_refs_only, &ibuf);
1342 done:
1343 TAILQ_FOREACH(pe, &have_refs, entry) {
1344 free((char *)pe->path);
1345 free(pe->data);
1347 got_pathlist_free(&have_refs);
1348 TAILQ_FOREACH(pe, &wanted_branches, entry)
1349 free((char *)pe->path);
1350 got_pathlist_free(&wanted_branches);
1351 if (fetchfd != -1 && close(fetchfd) == -1 && err == NULL)
1352 err = got_error_from_errno("close");
1353 if (packfd != -1 && close(packfd) == -1 && err == NULL)
1354 err = got_error_from_errno("close");
1355 if (err != NULL)
1356 got_privsep_send_error(&ibuf, err);
1357 else
1358 err = send_fetch_done(&ibuf, pack_sha1);
1359 if (err != NULL) {
1360 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1361 got_privsep_send_error(&ibuf, err);
1364 exit(0);