commit 39ff877fbb5335dfcf380e186a76b54074407112
from: Stefan Sperling <stsp@stsp.name>
date: Tue Mar 13 15:31:33 2018 UTC

expand deltas in memory if result size is < 32MB

commit - 55ef7a6f543a55f828618dc7fe6c7e9b842c355e
commit + 39ff877fbb5335dfcf380e186a76b54074407112
blob - 537dfbc7f21349731f5c0fee299fae0f71be5f33
blob + f8ab225fffcd98374fba89b150774262431fbb13
--- lib/delta.c
+++ lib/delta.c
@@ -233,6 +233,9 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 	uint64_t base_size, result_size;
 	size_t remain, outsize = 0;
 	const uint8_t *p;
+	FILE *memstream = NULL;
+	char *memstream_buf = NULL;
+	size_t memstream_size = 0;
 
 	if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
 		return got_error(GOT_ERR_BAD_DELTA);
@@ -251,6 +254,9 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 	if (err)
 		return err;
 
+	if (result_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
+		memstream = open_memstream(&memstream_buf, &memstream_size);
+
 	/* Decode and execute copy instructions from the delta stream. */
 	err = next_delta_byte(&p, &remain);
 	while (err == NULL && remain > 0) {
@@ -260,7 +266,8 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 			err = parse_opcode(&offset, &len, &p, &remain);
 			if (err)
 				break;
-			err = copy_from_base(base_file, offset, len, outfile);
+			err = copy_from_base(base_file, offset, len,
+			    memstream ? memstream : outfile);
 			if (err == NULL) {
 				outsize += len;
 				if (remain > 0) {
@@ -277,7 +284,8 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 			err = next_delta_byte(&p, &remain);
 			if (err)
 				break;
-			err = copy_from_delta(&p, &remain, len, outfile);
+			err = copy_from_delta(&p, &remain, len,
+			    memstream ? memstream : outfile);
 			if (err == NULL)
 				outsize += len;
 		}
@@ -286,6 +294,16 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 	if (outsize != result_size)
 		err = got_error(GOT_ERR_BAD_DELTA);
 
+	if (memstream != NULL) {
+		fclose(memstream);
+		if (err == NULL) {
+			size_t n;
+			n = fwrite(memstream_buf, 1, memstream_size, outfile);
+			if (n != memstream_size)
+				err = got_ferror(outfile, GOT_ERR_IO);
+		}
+		free(memstream_buf);
+	}
 	if (err == NULL)
 		rewind(outfile);
 	return err;
blob - e34bc99ac2cc963e73e3c9b78104062681407ccb
blob + c0738c0ca2196c671aa755529b9011445ce366e5
--- lib/got_delta_lib.h
+++ lib/got_delta_lib.h
@@ -38,6 +38,12 @@ const struct got_error *got_delta_apply(FILE *, const 
     FILE *);
 
 /*
+ * The amount of result data we may keep in RAM while applying deltas.
+ * Data larger than this is written to disk during delta application (slow).
+ */
+#define GOT_DELTA_RESULT_SIZE_CACHED_MAX	(32 * 1024 * 1024) /* bytes */
+
+/*
  * Definitions for delta data streams.
  */