Blame

Date:
Message:
map raw object files into memory while packing if possible
Actions:
History | Blob | Raw File
  1 2181e0c8 2019-03-19 stsp /*

  2 2181e0c8 2019-03-19 stsp  * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>

  3 2181e0c8 2019-03-19 stsp  *

  4 2181e0c8 2019-03-19 stsp  * Permission to use, copy, modify, and distribute this software for any

  5 2181e0c8 2019-03-19 stsp  * purpose with or without fee is hereby granted, provided that the above

  6 2181e0c8 2019-03-19 stsp  * copyright notice and this permission notice appear in all copies.

  7 2181e0c8 2019-03-19 stsp  *

  8 2181e0c8 2019-03-19 stsp  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES

  9 2181e0c8 2019-03-19 stsp  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF

 10 2181e0c8 2019-03-19 stsp  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR

 11 2181e0c8 2019-03-19 stsp  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

 12 2181e0c8 2019-03-19 stsp  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN

 13 2181e0c8 2019-03-19 stsp  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF

 14 2181e0c8 2019-03-19 stsp  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

 15 2181e0c8 2019-03-19 stsp  */

 16 2181e0c8 2019-03-19 stsp 

 17 2181e0c8 2019-03-19 stsp #include <sys/queue.h>

 18 2181e0c8 2019-03-19 stsp 

 19 2181e0c8 2019-03-19 stsp #include <errno.h>

 20 2181e0c8 2019-03-19 stsp #include <stdio.h>

 21 2181e0c8 2019-03-19 stsp #include <stdlib.h>

 22 2181e0c8 2019-03-19 stsp #include <string.h>

 23 2181e0c8 2019-03-19 stsp #include <sha1.h>

 24 2181e0c8 2019-03-19 stsp #include <zlib.h>

 25 2181e0c8 2019-03-19 stsp #include <time.h>

 26 2181e0c8 2019-03-19 stsp 

 27 2181e0c8 2019-03-19 stsp #include "got_error.h"

 28 2181e0c8 2019-03-19 stsp #include "got_object.h"

 29 324d37e7 2019-05-11 stsp #include "got_path.h"

 30 2181e0c8 2019-03-19 stsp 

 31 2181e0c8 2019-03-19 stsp #include "got_lib_deflate.h"

 32 2181e0c8 2019-03-19 stsp 

 33 2181e0c8 2019-03-19 stsp #ifndef MIN

 34 2181e0c8 2019-03-19 stsp #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))

 35 2181e0c8 2019-03-19 stsp #endif

 36 2181e0c8 2019-03-19 stsp 

 37 2181e0c8 2019-03-19 stsp const struct got_error *

 38 3b9e6fcf 2021-06-05 stsp got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)

 39 2181e0c8 2019-03-19 stsp {

 40 2181e0c8 2019-03-19 stsp 	const struct got_error *err = NULL;

 41 2181e0c8 2019-03-19 stsp 	int zerr;

 42 2181e0c8 2019-03-19 stsp 

 43 2181e0c8 2019-03-19 stsp 	memset(&zb->z, 0, sizeof(zb->z));

 44 2181e0c8 2019-03-19 stsp 

 45 2181e0c8 2019-03-19 stsp 	zb->z.zalloc = Z_NULL;

 46 2181e0c8 2019-03-19 stsp 	zb->z.zfree = Z_NULL;

 47 2181e0c8 2019-03-19 stsp 	zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);

 48 2181e0c8 2019-03-19 stsp 	if (zerr != Z_OK) {

 49 2181e0c8 2019-03-19 stsp 		if  (zerr == Z_ERRNO)

 50 638f9024 2019-05-13 stsp 			return got_error_from_errno("deflateInit");

 51 2181e0c8 2019-03-19 stsp 		if  (zerr == Z_MEM_ERROR) {

 52 2181e0c8 2019-03-19 stsp 			errno = ENOMEM;

 53 638f9024 2019-05-13 stsp 			return got_error_from_errno("deflateInit");

 54 2181e0c8 2019-03-19 stsp 		}

 55 2181e0c8 2019-03-19 stsp 		return got_error(GOT_ERR_COMPRESSION);

 56 2181e0c8 2019-03-19 stsp 	}

 57 2181e0c8 2019-03-19 stsp 

 58 2181e0c8 2019-03-19 stsp 	zb->inlen = zb->outlen = bufsize;

 59 2181e0c8 2019-03-19 stsp 

 60 2181e0c8 2019-03-19 stsp 	zb->inbuf = calloc(1, zb->inlen);

 61 2181e0c8 2019-03-19 stsp 	if (zb->inbuf == NULL) {

 62 638f9024 2019-05-13 stsp 		err = got_error_from_errno("calloc");

 63 2181e0c8 2019-03-19 stsp 		goto done;

 64 2181e0c8 2019-03-19 stsp 	}

 65 2181e0c8 2019-03-19 stsp 

 66 2181e0c8 2019-03-19 stsp 	zb->flags = 0;

 67 2181e0c8 2019-03-19 stsp 	if (outbuf == NULL) {

 68 2181e0c8 2019-03-19 stsp 		zb->outbuf = calloc(1, zb->outlen);

 69 2181e0c8 2019-03-19 stsp 		if (zb->outbuf == NULL) {

 70 638f9024 2019-05-13 stsp 			err = got_error_from_errno("calloc");

 71 2181e0c8 2019-03-19 stsp 			goto done;

 72 2181e0c8 2019-03-19 stsp 		}

 73 2181e0c8 2019-03-19 stsp 		zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;

 74 2181e0c8 2019-03-19 stsp 	} else

 75 2181e0c8 2019-03-19 stsp 		zb->outbuf = outbuf;

 76 2181e0c8 2019-03-19 stsp done:

 77 2181e0c8 2019-03-19 stsp 	if (err)

 78 2181e0c8 2019-03-19 stsp 		got_deflate_end(zb);

 79 2181e0c8 2019-03-19 stsp 	return err;

 80 2181e0c8 2019-03-19 stsp }

 81 2181e0c8 2019-03-19 stsp 

 82 91b40e30 2021-05-21 stsp static void

 83 31e61ec1 2021-09-28 naddy csum_output(struct got_deflate_checksum *csum, const uint8_t *buf, size_t len)

 84 91b40e30 2021-05-21 stsp {

 85 91b40e30 2021-05-21 stsp 	if (csum->output_crc)

 86 91b40e30 2021-05-21 stsp 		*csum->output_crc = crc32(*csum->output_crc, buf, len);

 87 91b40e30 2021-05-21 stsp 

 88 91b40e30 2021-05-21 stsp 	if (csum->output_sha1)

 89 91b40e30 2021-05-21 stsp 		SHA1Update(csum->output_sha1, buf, len);

 90 91b40e30 2021-05-21 stsp }

 91 91b40e30 2021-05-21 stsp 

 92 2181e0c8 2019-03-19 stsp const struct got_error *

 93 2181e0c8 2019-03-19 stsp got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)

 94 2181e0c8 2019-03-19 stsp {

 95 2181e0c8 2019-03-19 stsp 	size_t last_total_out = zb->z.total_out;

 96 2181e0c8 2019-03-19 stsp 	z_stream *z = &zb->z;

 97 2181e0c8 2019-03-19 stsp 	int ret = Z_ERRNO;

 98 2181e0c8 2019-03-19 stsp 

 99 2181e0c8 2019-03-19 stsp 	z->next_out = zb->outbuf;

100 2181e0c8 2019-03-19 stsp 	z->avail_out = zb->outlen;

101 2181e0c8 2019-03-19 stsp 

102 2181e0c8 2019-03-19 stsp 	*outlenp = 0;

103 2181e0c8 2019-03-19 stsp 	do {

104 2181e0c8 2019-03-19 stsp 		if (z->avail_in == 0) {

105 2181e0c8 2019-03-19 stsp 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);

106 2181e0c8 2019-03-19 stsp 			if (n == 0) {

107 2181e0c8 2019-03-19 stsp 				if (ferror(f))

108 2181e0c8 2019-03-19 stsp 					return got_ferror(f, GOT_ERR_IO);

109 2181e0c8 2019-03-19 stsp 				/* EOF */

110 4e4a7005 2019-04-13 stsp 				ret = deflate(z, Z_FINISH);

111 2181e0c8 2019-03-19 stsp 				break;

112 2181e0c8 2019-03-19 stsp 			}

113 2181e0c8 2019-03-19 stsp 			z->next_in = zb->inbuf;

114 2181e0c8 2019-03-19 stsp 			z->avail_in = n;

115 2181e0c8 2019-03-19 stsp 		}

116 2181e0c8 2019-03-19 stsp 		ret = deflate(z, Z_NO_FLUSH);

117 2181e0c8 2019-03-19 stsp 	} while (ret == Z_OK && z->avail_out > 0);

118 2181e0c8 2019-03-19 stsp 

119 2181e0c8 2019-03-19 stsp 	if (ret == Z_OK) {

120 2181e0c8 2019-03-19 stsp 		zb->flags |= GOT_DEFLATE_F_HAVE_MORE;

121 2181e0c8 2019-03-19 stsp 	} else {

122 2181e0c8 2019-03-19 stsp 		if (ret != Z_STREAM_END)

123 2181e0c8 2019-03-19 stsp 			return got_error(GOT_ERR_COMPRESSION);

124 2181e0c8 2019-03-19 stsp 		zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;

125 2181e0c8 2019-03-19 stsp 	}

126 2181e0c8 2019-03-19 stsp 

127 2181e0c8 2019-03-19 stsp 	*outlenp = z->total_out - last_total_out;

128 2181e0c8 2019-03-19 stsp 	return NULL;

129 2181e0c8 2019-03-19 stsp }

130 2181e0c8 2019-03-19 stsp 

131 64a8571e 2022-01-07 stsp const struct got_error *

132 64a8571e 2022-01-07 stsp got_deflate_read_mmap(struct got_deflate_buf *zb, uint8_t *map, size_t offset,

133 64a8571e 2022-01-07 stsp     size_t len, size_t *outlenp, size_t *consumed)

134 64a8571e 2022-01-07 stsp {

135 64a8571e 2022-01-07 stsp 	z_stream *z = &zb->z;

136 64a8571e 2022-01-07 stsp 	size_t last_total_out = z->total_out;

137 64a8571e 2022-01-07 stsp 	int ret = Z_ERRNO;

138 64a8571e 2022-01-07 stsp 

139 64a8571e 2022-01-07 stsp 	z->next_out = zb->outbuf;

140 64a8571e 2022-01-07 stsp 	z->avail_out = zb->outlen;

141 64a8571e 2022-01-07 stsp 

142 64a8571e 2022-01-07 stsp 	*outlenp = 0;

143 64a8571e 2022-01-07 stsp 	*consumed = 0;

144 64a8571e 2022-01-07 stsp 	do {

145 64a8571e 2022-01-07 stsp 		size_t last_total_in = z->total_in;

146 64a8571e 2022-01-07 stsp 		if (z->avail_in == 0) {

147 64a8571e 2022-01-07 stsp 			z->next_in = map + offset + *consumed;

148 64a8571e 2022-01-07 stsp 			z->avail_in = len - *consumed;

149 64a8571e 2022-01-07 stsp 			if (z->avail_in == 0) {

150 64a8571e 2022-01-07 stsp 				/* EOF */

151 64a8571e 2022-01-07 stsp 				ret = deflate(z, Z_FINISH);

152 64a8571e 2022-01-07 stsp 				break;

153 64a8571e 2022-01-07 stsp 			}

154 64a8571e 2022-01-07 stsp 		}

155 64a8571e 2022-01-07 stsp 		ret = deflate(z, Z_NO_FLUSH);

156 64a8571e 2022-01-07 stsp 		*consumed += z->total_in - last_total_in;

157 64a8571e 2022-01-07 stsp 	} while (ret == Z_OK && z->avail_out > 0);

158 64a8571e 2022-01-07 stsp 

159 64a8571e 2022-01-07 stsp 	if (ret == Z_OK) {

160 64a8571e 2022-01-07 stsp 		zb->flags |= GOT_DEFLATE_F_HAVE_MORE;

161 64a8571e 2022-01-07 stsp 	} else {

162 64a8571e 2022-01-07 stsp 		if (ret != Z_STREAM_END)

163 64a8571e 2022-01-07 stsp 			return got_error(GOT_ERR_COMPRESSION);

164 64a8571e 2022-01-07 stsp 		zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;

165 64a8571e 2022-01-07 stsp 	}

166 64a8571e 2022-01-07 stsp 

167 64a8571e 2022-01-07 stsp 	*outlenp = z->total_out - last_total_out;

168 64a8571e 2022-01-07 stsp 	return NULL;

169 64a8571e 2022-01-07 stsp }

170 64a8571e 2022-01-07 stsp 

171 2181e0c8 2019-03-19 stsp void

172 2181e0c8 2019-03-19 stsp got_deflate_end(struct got_deflate_buf *zb)

173 2181e0c8 2019-03-19 stsp {

174 2181e0c8 2019-03-19 stsp 	free(zb->inbuf);

175 2181e0c8 2019-03-19 stsp 	if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)

176 2181e0c8 2019-03-19 stsp 		free(zb->outbuf);

177 2181e0c8 2019-03-19 stsp 	deflateEnd(&zb->z);

178 2181e0c8 2019-03-19 stsp }

179 2181e0c8 2019-03-19 stsp 

180 2181e0c8 2019-03-19 stsp const struct got_error *

181 91b40e30 2021-05-21 stsp got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile,

182 91b40e30 2021-05-21 stsp     struct got_deflate_checksum *csum)

183 2181e0c8 2019-03-19 stsp {

184 2181e0c8 2019-03-19 stsp 	const struct got_error *err;

185 2181e0c8 2019-03-19 stsp 	size_t avail;

186 2181e0c8 2019-03-19 stsp 	struct got_deflate_buf zb;

187 2181e0c8 2019-03-19 stsp 

188 3b9e6fcf 2021-06-05 stsp 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);

189 2181e0c8 2019-03-19 stsp 	if (err)

190 2181e0c8 2019-03-19 stsp 		goto done;

191 2181e0c8 2019-03-19 stsp 

192 2181e0c8 2019-03-19 stsp 	*outlen = 0;

193 2181e0c8 2019-03-19 stsp 

194 2181e0c8 2019-03-19 stsp 	do {

195 2181e0c8 2019-03-19 stsp 		err = got_deflate_read(&zb, infile, &avail);

196 2181e0c8 2019-03-19 stsp 		if (err)

197 2181e0c8 2019-03-19 stsp 			goto done;

198 2181e0c8 2019-03-19 stsp 		if (avail > 0) {

199 2181e0c8 2019-03-19 stsp 			size_t n;

200 2181e0c8 2019-03-19 stsp 			n = fwrite(zb.outbuf, avail, 1, outfile);

201 2181e0c8 2019-03-19 stsp 			if (n != 1) {

202 2181e0c8 2019-03-19 stsp 				err = got_ferror(outfile, GOT_ERR_IO);

203 2181e0c8 2019-03-19 stsp 				goto done;

204 2181e0c8 2019-03-19 stsp 			}

205 3b9e6fcf 2021-06-05 stsp 			if (csum)

206 3b9e6fcf 2021-06-05 stsp 				csum_output(csum, zb.outbuf, avail);

207 2181e0c8 2019-03-19 stsp 			*outlen += avail;

208 2181e0c8 2019-03-19 stsp 		}

209 2181e0c8 2019-03-19 stsp 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);

210 2181e0c8 2019-03-19 stsp 

211 2181e0c8 2019-03-19 stsp done:

212 2181e0c8 2019-03-19 stsp 	got_deflate_end(&zb);

213 2181e0c8 2019-03-19 stsp 	return err;

214 2181e0c8 2019-03-19 stsp }

215 64a8571e 2022-01-07 stsp 

216 64a8571e 2022-01-07 stsp const struct got_error *

217 64a8571e 2022-01-07 stsp got_deflate_to_file_mmap(size_t *outlen, uint8_t *map, size_t offset,

218 64a8571e 2022-01-07 stsp     size_t len, FILE *outfile, struct got_deflate_checksum *csum)

219 64a8571e 2022-01-07 stsp {

220 64a8571e 2022-01-07 stsp 	const struct got_error *err;

221 64a8571e 2022-01-07 stsp 	size_t avail, consumed;

222 64a8571e 2022-01-07 stsp 	struct got_deflate_buf zb;

223 64a8571e 2022-01-07 stsp 

224 64a8571e 2022-01-07 stsp 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);

225 64a8571e 2022-01-07 stsp 	if (err)

226 64a8571e 2022-01-07 stsp 		goto done;

227 64a8571e 2022-01-07 stsp 

228 64a8571e 2022-01-07 stsp 	*outlen = 0;

229 64a8571e 2022-01-07 stsp 	do {

230 64a8571e 2022-01-07 stsp 		err = got_deflate_read_mmap(&zb, map, offset, len, &avail,

231 64a8571e 2022-01-07 stsp 		    &consumed);

232 64a8571e 2022-01-07 stsp 		if (err)

233 64a8571e 2022-01-07 stsp 			goto done;

234 64a8571e 2022-01-07 stsp 		offset += consumed;

235 64a8571e 2022-01-07 stsp 		len -= consumed;

236 64a8571e 2022-01-07 stsp 		if (avail > 0) {

237 64a8571e 2022-01-07 stsp 			size_t n;

238 64a8571e 2022-01-07 stsp 			n = fwrite(zb.outbuf, avail, 1, outfile);

239 64a8571e 2022-01-07 stsp 			if (n != 1) {

240 64a8571e 2022-01-07 stsp 				err = got_ferror(outfile, GOT_ERR_IO);

241 64a8571e 2022-01-07 stsp 				goto done;

242 64a8571e 2022-01-07 stsp 			}

243 64a8571e 2022-01-07 stsp 			if (csum)

244 64a8571e 2022-01-07 stsp 				csum_output(csum, zb.outbuf, avail);

245 64a8571e 2022-01-07 stsp 			*outlen += avail;

246 64a8571e 2022-01-07 stsp 		}

247 64a8571e 2022-01-07 stsp 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);

248 64a8571e 2022-01-07 stsp 

249 64a8571e 2022-01-07 stsp done:

250 64a8571e 2022-01-07 stsp 	got_deflate_end(&zb);

251 64a8571e 2022-01-07 stsp 	return err;

252 64a8571e 2022-01-07 stsp }