2 * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <sys/queue.h>
29 #include "got_error.h"
30 #include "got_object.h"
32 #include "got_lib_path.h"
33 #include "got_lib_fileindex.h"
35 struct got_fileindex {
36 struct got_fileindex_tree entries;
38 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
41 const struct got_error *
42 got_fileindex_entry_update(struct got_fileindex_entry *entry,
43 const char *ondisk_path, uint8_t *blob_sha1, uint8_t *commit_sha1)
47 if (lstat(ondisk_path, &sb) != 0)
48 return got_error_from_errno();
50 entry->ctime_sec = sb.st_ctime;
51 entry->ctime_nsec = sb.st_ctimensec;
52 entry->mtime_sec = sb.st_mtime;
53 entry->mtime_nsec = sb.st_mtimensec;
54 entry->uid = sb.st_uid;
55 entry->gid = sb.st_gid;
56 entry->size = (sb.st_size & 0xffffffff);
57 if (sb.st_mode & S_IFLNK)
58 entry->mode = GOT_FILEIDX_MODE_SYMLINK;
60 entry->mode = GOT_FILEIDX_MODE_REGULAR_FILE;
61 entry->mode |= ((sb.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) <<
62 GOT_FILEIDX_MODE_PERMS_SHIFT);
63 memcpy(entry->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
64 memcpy(entry->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
69 const struct got_error *
70 got_fileindex_entry_alloc(struct got_fileindex_entry **entry,
71 const char *ondisk_path, const char *relpath, uint8_t *blob_sha1,
76 *entry = calloc(1, sizeof(**entry));
78 return got_error_from_errno();
80 (*entry)->path = strdup(relpath);
81 if ((*entry)->path == NULL) {
82 const struct got_error *err = got_error_from_errno();
88 len = strlen(relpath);
89 if (len > GOT_FILEIDX_F_PATH_LEN)
90 len = GOT_FILEIDX_F_PATH_LEN;
91 (*entry)->flags |= len;
93 return got_fileindex_entry_update(*entry, ondisk_path, blob_sha1,
98 got_fileindex_entry_free(struct got_fileindex_entry *entry)
104 static const struct got_error *
105 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *entry)
107 if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
108 return got_error(GOT_ERR_NO_SPACE);
110 RB_INSERT(got_fileindex_tree, &fileindex->entries, entry);
111 fileindex->nentries++;
115 const struct got_error *
116 got_fileindex_entry_add(struct got_fileindex *fileindex,
117 struct got_fileindex_entry *entry)
119 /* Flag this entry until it gets written out to disk. */
120 entry->flags |= GOT_FILEIDX_F_INTENT_TO_ADD;
122 return add_entry(fileindex, entry);
126 got_fileindex_entry_remove(struct got_fileindex *fileindex,
127 struct got_fileindex_entry *entry)
129 RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
130 fileindex->nentries--;
133 struct got_fileindex_entry *
134 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path)
136 struct got_fileindex_entry key;
137 memset(&key, 0, sizeof(key));
138 key.path = (char *)path;
139 return RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
142 const struct got_error *
143 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
144 got_fileindex_cb cb, void *cb_arg)
146 const struct got_error *err;
147 struct got_fileindex_entry *entry, *tmp;
149 RB_FOREACH_SAFE(entry, got_fileindex_tree, &fileindex->entries, tmp) {
150 err = (*cb)(cb_arg, entry);
157 struct got_fileindex *
158 got_fileindex_alloc(void)
160 struct got_fileindex *fileindex;
162 fileindex = calloc(1, sizeof(*fileindex));
163 if (fileindex == NULL)
166 RB_INIT(&fileindex->entries);
171 got_fileindex_free(struct got_fileindex *fileindex)
173 struct got_fileindex_entry *entry;
175 while ((entry = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
176 RB_REMOVE(got_fileindex_tree, &fileindex->entries, entry);
177 got_fileindex_entry_free(entry);
182 static const struct got_error *
183 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
188 SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
189 n = fwrite(&val, 1, sizeof(val), outfile);
190 if (n != sizeof(val))
191 return got_ferror(outfile, GOT_ERR_IO);
195 static const struct got_error *
196 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
201 SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
202 n = fwrite(&val, 1, sizeof(val), outfile);
203 if (n != sizeof(val))
204 return got_ferror(outfile, GOT_ERR_IO);
208 static const struct got_error *
209 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
214 SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
215 n = fwrite(&val, 1, sizeof(val), outfile);
216 if (n != sizeof(val))
217 return got_ferror(outfile, GOT_ERR_IO);
221 static const struct got_error *
222 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
224 size_t n, len, pad = 0;
225 static const uint8_t zero[8] = { 0 };
228 while ((len + pad) % 8 != 0)
231 pad = 8; /* NUL-terminate */
233 SHA1Update(ctx, path, len);
234 n = fwrite(path, 1, len, outfile);
236 return got_ferror(outfile, GOT_ERR_IO);
237 SHA1Update(ctx, zero, pad);
238 n = fwrite(zero, 1, pad, outfile);
240 return got_ferror(outfile, GOT_ERR_IO);
244 static const struct got_error *
245 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *entry,
248 const struct got_error *err;
251 err = write_fileindex_val64(ctx, entry->ctime_sec, outfile);
254 err = write_fileindex_val64(ctx, entry->ctime_nsec, outfile);
257 err = write_fileindex_val64(ctx, entry->mtime_sec, outfile);
260 err = write_fileindex_val64(ctx, entry->mtime_nsec, outfile);
264 err = write_fileindex_val32(ctx, entry->uid, outfile);
267 err = write_fileindex_val32(ctx, entry->gid, outfile);
270 err = write_fileindex_val32(ctx, entry->size, outfile);
274 err = write_fileindex_val16(ctx, entry->mode, outfile);
278 SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
279 n = fwrite(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
280 if (n != SHA1_DIGEST_LENGTH)
281 return got_ferror(outfile, GOT_ERR_IO);
283 SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
284 n = fwrite(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
285 if (n != SHA1_DIGEST_LENGTH)
286 return got_ferror(outfile, GOT_ERR_IO);
288 err = write_fileindex_val32(ctx, entry->flags, outfile);
292 err = write_fileindex_path(ctx, entry->path, outfile);
296 const struct got_error *
297 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
299 const struct got_error *err = NULL;
300 struct got_fileindex_hdr hdr;
302 uint8_t sha1[SHA1_DIGEST_LENGTH];
304 struct got_fileindex_entry *entry;
308 hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
309 hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
310 hdr.nentries = htobe32(fileindex->nentries);
312 SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
313 SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
314 SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
315 n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
316 if (n != sizeof(hdr.signature))
317 return got_ferror(outfile, GOT_ERR_IO);
318 n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
319 if (n != sizeof(hdr.version))
320 return got_ferror(outfile, GOT_ERR_IO);
321 n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
322 if (n != sizeof(hdr.nentries))
323 return got_ferror(outfile, GOT_ERR_IO);
325 RB_FOREACH(entry, got_fileindex_tree, &fileindex->entries) {
326 entry->flags &= ~GOT_FILEIDX_F_INTENT_TO_ADD;
327 err = write_fileindex_entry(&ctx, entry, outfile);
332 SHA1Final(sha1, &ctx);
333 n = fwrite(sha1, 1, sizeof(sha1), outfile);
334 if (n != sizeof(sha1))
335 return got_ferror(outfile, GOT_ERR_IO);
337 if (fflush(outfile) != 0)
338 return got_error_from_errno();
343 static const struct got_error *
344 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
348 n = fread(val, 1, sizeof(*val), infile);
349 if (n != sizeof(*val))
350 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
351 SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
352 *val = be64toh(*val);
356 static const struct got_error *
357 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
361 n = fread(val, 1, sizeof(*val), infile);
362 if (n != sizeof(*val))
363 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
364 SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
365 *val = be32toh(*val);
369 static const struct got_error *
370 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
374 n = fread(val, 1, sizeof(*val), infile);
375 if (n != sizeof(*val))
376 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
377 SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
378 *val = be16toh(*val);
382 static const struct got_error *
383 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
385 const struct got_error *err = NULL;
387 size_t n, len = 0, totlen = sizeof(buf);
389 *path = malloc(totlen);
391 return got_error_from_errno();
394 n = fread(buf, 1, sizeof(buf), infile);
395 if (n != sizeof(buf))
396 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
397 if (len + sizeof(buf) > totlen) {
398 char *p = reallocarray(*path, totlen + sizeof(buf), 1);
400 err = got_error_from_errno();
403 totlen += sizeof(buf);
406 SHA1Update(ctx, buf, sizeof(buf));
407 memcpy(*path + len, buf, sizeof(buf));
409 } while (memchr(buf, '\0', sizeof(buf)) == NULL);
418 static const struct got_error *
419 read_fileindex_entry(struct got_fileindex_entry **entryp, SHA1_CTX *ctx,
422 const struct got_error *err;
423 struct got_fileindex_entry *entry;
428 entry = calloc(1, sizeof(*entry));
430 return got_error_from_errno();
432 err = read_fileindex_val64(&entry->ctime_sec, ctx, infile);
435 err = read_fileindex_val64(&entry->ctime_nsec, ctx, infile);
438 err = read_fileindex_val64(&entry->mtime_sec, ctx, infile);
441 err = read_fileindex_val64(&entry->mtime_nsec, ctx, infile);
445 err = read_fileindex_val32(&entry->uid, ctx, infile);
448 err = read_fileindex_val32(&entry->gid, ctx, infile);
451 err = read_fileindex_val32(&entry->size, ctx, infile);
455 err = read_fileindex_val16(&entry->mode, ctx, infile);
459 n = fread(entry->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
460 if (n != SHA1_DIGEST_LENGTH) {
461 err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
464 SHA1Update(ctx, entry->blob_sha1, SHA1_DIGEST_LENGTH);
466 n = fread(entry->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
467 if (n != SHA1_DIGEST_LENGTH) {
468 err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
471 SHA1Update(ctx, entry->commit_sha1, SHA1_DIGEST_LENGTH);
473 err = read_fileindex_val32(&entry->flags, ctx, infile);
477 err = read_fileindex_path(&entry->path, ctx, infile);
486 const struct got_error *
487 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
489 const struct got_error *err = NULL;
490 struct got_fileindex_hdr hdr;
492 struct got_fileindex_entry *entry;
493 uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
494 uint8_t sha1[SHA1_DIGEST_LENGTH];
500 n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
501 if (n != sizeof(hdr.signature)) {
502 if (n == 0) /* EOF */
504 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
506 n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
507 if (n != sizeof(hdr.version)) {
508 if (n == 0) /* EOF */
510 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
512 n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
513 if (n != sizeof(hdr.nentries)) {
514 if (n == 0) /* EOF */
516 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
519 SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
520 SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
521 SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
523 hdr.signature = be32toh(hdr.signature);
524 hdr.version = be32toh(hdr.version);
525 hdr.nentries = be32toh(hdr.nentries);
527 if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
528 return got_error(GOT_ERR_FILEIDX_SIG);
529 if (hdr.version != GOT_FILE_INDEX_VERSION)
530 return got_error(GOT_ERR_FILEIDX_VER);
532 for (i = 0; i < hdr.nentries; i++) {
533 err = read_fileindex_entry(&entry, &ctx, infile);
536 err = add_entry(fileindex, entry);
541 n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
542 if (n != sizeof(sha1_expected))
543 return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
544 SHA1Final(sha1, &ctx);
545 if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
546 return got_error(GOT_ERR_FILEIDX_CSUM);
551 struct got_fileindex_entry *
552 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
554 struct got_fileindex_entry *next;
556 next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
558 /* Skip entries which were newly added by diff callbacks. */
559 while (next && (next->flags & GOT_FILEIDX_F_INTENT_TO_ADD))
560 next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
565 static const struct got_error *
566 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **,
567 struct got_tree_object *, const char *, struct got_repository *,
568 struct got_fileindex_diff_tree_cb *, void *);
570 static const struct got_error *
571 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
572 struct got_fileindex_entry **ie, struct got_tree_entry *te,
573 const char *path, struct got_repository *repo,
574 struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
576 const struct got_error *err = NULL;
578 if (S_ISDIR(te->mode)) {
580 struct got_tree_object *subtree;
582 if (asprintf(&subpath, "%s%s%s", path,
583 path[0] == '\0' ? "" : "/", te->name) == -1)
584 return got_error_from_errno();
586 err = got_object_open_as_tree(&subtree, repo, te->id);
592 err = diff_fileindex_tree(fileindex, ie, subtree,
593 subpath, repo, cb, cb_arg);
595 got_object_tree_close(subtree);
600 *next = SIMPLEQ_NEXT(te, entry);
605 * Decide whether a fileindex entry path is equivalent to a tree entry path,
606 * and if it is not, then decide which of the two should be processed first.
609 cmp_entries(const char *ie_path, const char *parent_path,
610 size_t parent_len, const char *te_name)
612 int cmp = strncmp(ie_path, parent_path, parent_len);
614 const char *ie_name = ie_path + parent_len;
615 while (ie_name[0] == '/')
617 cmp = strcmp(ie_name, te_name);
623 static const struct got_error *
624 diff_fileindex_tree(struct got_fileindex *fileindex,
625 struct got_fileindex_entry **ie, struct got_tree_object *tree,
626 const char *path, struct got_repository *repo,
627 struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
629 const struct got_error *err = NULL;
630 struct got_tree_entry *te = NULL;
631 size_t path_len = strlen(path);
632 const struct got_tree_entries *entries;
633 struct got_fileindex_entry *next;
635 entries = got_object_tree_get_entries(tree);
636 te = SIMPLEQ_FIRST(&entries->head);
639 int cmp = cmp_entries((*ie)->path, path, path_len,
642 err = cb->diff_old_new(cb_arg, *ie, te,
646 *ie = walk_fileindex(fileindex, *ie);
647 err = walk_tree(&te, fileindex, ie, te,
648 path, repo, cb, cb_arg);
649 } else if (cmp < 0 ) {
650 next = walk_fileindex(fileindex, *ie);
651 err = cb->diff_old(cb_arg, *ie, path);
656 err = cb->diff_new(cb_arg, te, path);
659 err = walk_tree(&te, fileindex, ie, te,
660 path, repo, cb, cb_arg);
665 next = walk_fileindex(fileindex, *ie);
666 err = cb->diff_old(cb_arg, *ie, path);
671 err = cb->diff_new(cb_arg, te, path);
674 err = walk_tree(&te, fileindex, ie, te, path, repo, cb,
679 } while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te);
684 const struct got_error *
685 got_fileindex_diff_tree(struct got_fileindex *fileindex,
686 struct got_tree_object *tree, struct got_repository *repo,
687 struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
689 struct got_fileindex_entry *min;
690 min = RB_MIN(got_fileindex_tree, &fileindex->entries);
691 return diff_fileindex_tree(fileindex, &min, tree, "", repo, cb, cb_arg);
694 static const struct got_error *
695 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **, DIR *,
696 const char *, struct got_repository *, struct got_fileindex_diff_dir_cb *,
699 static const struct got_error *
700 walk_dir(struct dirent **next, struct got_fileindex *fileindex,
701 struct got_fileindex_entry **ie, struct dirent *de, const char *path,
702 DIR *dir, struct got_repository *repo,
703 struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
705 const struct got_error *err = NULL;
707 if (de->d_type == DT_DIR) {
711 if (asprintf(&subpath, "%s%s%s", path,
712 path[0] == '\0' ? "" : "/", de->d_name) == -1)
713 return got_error_from_errno();
715 subdir = opendir(subpath);
716 if (subdir == NULL) {
718 return got_error_from_errno();
721 err = diff_fileindex_dir(fileindex, ie, subdir, subpath, repo,
729 *next = readdir(dir);
733 static const struct got_error *
734 diff_fileindex_dir(struct got_fileindex *fileindex,
735 struct got_fileindex_entry **ie, DIR *dir, const char *path,
736 struct got_repository *repo, struct got_fileindex_diff_dir_cb *cb,
739 const struct got_error *err = NULL;
740 struct dirent *de = NULL;
741 size_t path_len = strlen(path);
742 struct got_fileindex_entry *next;
746 if (strcmp(de->d_name, ".") == 0 ||
747 strcmp(de->d_name, "..") == 0) {
752 int cmp = cmp_entries((*ie)->path, path, path_len,
755 err = cb->diff_old_new(cb_arg, *ie, de, path);
758 *ie = walk_fileindex(fileindex, *ie);
759 err = walk_dir(&de, fileindex, ie, de, path,
760 dir, repo, cb, cb_arg);
761 } else if (cmp < 0 ) {
762 next = walk_fileindex(fileindex, *ie);
763 err = cb->diff_old(cb_arg, *ie, path);
768 err = cb->diff_new(cb_arg, de, path);
771 err = walk_dir(&de, fileindex, ie, de, path,
772 dir, repo, cb, cb_arg);
777 } while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || de);
782 const struct got_error *
783 got_fileindex_diff_dir(struct got_fileindex *fileindex, DIR *dir,
784 struct got_repository *repo, struct got_fileindex_diff_dir_cb *cb,
787 struct got_fileindex_entry *min;
788 min = RB_MIN(got_fileindex_tree, &fileindex->entries);
789 return diff_fileindex_dir(fileindex, &min, dir, "", repo, cb, cb_arg);
792 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);