1 --- test021.left.txt
2 +++ test021.right.txt
3 @@ -1,4 +1,4 @@
4 -/* $OpenBSD: softraid_crypto.c,v 1.91 2013/03/31 15:44:52 jsing Exp $ */
5 +/* $OpenBSD: softraid_crypto.c,v 1.139 2020/07/13 00:06:22 kn Exp $ */
6  /*
7   * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
8   * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
9 @@ -25,7 +25,6 @@
10  #include <sys/buf.h>
11  #include <sys/device.h>
12  #include <sys/ioctl.h>
13 -#include <sys/proc.h>
14  #include <sys/malloc.h>
15  #include <sys/pool.h>
16  #include <sys/kernel.h>
17 @@ -34,6 +33,7 @@
18  #include <sys/queue.h>
19  #include <sys/fcntl.h>
20  #include <sys/disklabel.h>
21 +#include <sys/vnode.h>
22  #include <sys/mount.h>
23  #include <sys/sensors.h>
24  #include <sys/stat.h>
25 @@ -42,7 +42,6 @@
26  #include <sys/dkio.h>
27  
28  #include <crypto/cryptodev.h>
29 -#include <crypto/cryptosoft.h>
30  #include <crypto/rijndael.h>
31  #include <crypto/md5.h>
32  #include <crypto/sha1.h>
33 @@ -54,7 +53,6 @@
34  #include <scsi/scsi_disk.h>
35  
36  #include <dev/softraidvar.h>
37 -#include <dev/rndvar.h>
38  
39  /*
40   * The per-I/O data that we need to preallocate. We cannot afford to allow I/O
41 @@ -62,18 +60,15 @@
42   * because we assert that only one ccb per WU will ever be active.
43   */
44  struct sr_crypto_wu {
45 -	TAILQ_ENTRY(sr_crypto_wu)	 cr_link;
46 +	struct sr_workunit		 cr_wu;		/* Must be first. */
47  	struct uio			 cr_uio;
48  	struct iovec			 cr_iov;
49  	struct cryptop	 		*cr_crp;
50 -	struct cryptodesc		*cr_descs;
51 -	struct sr_workunit		*cr_wu;
52  	void				*cr_dmabuf;
53  };
54  
55  
56 -struct sr_crypto_wu *sr_crypto_wu_get(struct sr_workunit *, int);
57 -void		sr_crypto_wu_put(struct sr_crypto_wu *);
58 +struct sr_crypto_wu *sr_crypto_prepare(struct sr_workunit *, int);
59  int		sr_crypto_create_keys(struct sr_discipline *);
60  int		sr_crypto_get_kdf(struct bioc_createraid *,
61  		    struct sr_discipline *);
62 @@ -92,12 +87,11 @@
63  		    struct bioc_discipline *);
64  int		sr_crypto_meta_opt_handler(struct sr_discipline *,
65  		    struct sr_meta_opt_hdr *);
66 -int		sr_crypto_write(struct cryptop *);
67 +void		sr_crypto_write(struct cryptop *);
68  int		sr_crypto_rw(struct sr_workunit *);
69 -int		sr_crypto_rw2(struct sr_workunit *, struct sr_crypto_wu *);
70 +int		sr_crypto_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
71  void		sr_crypto_done(struct sr_workunit *);
72 -int		sr_crypto_read(struct cryptop *);
73 -void		sr_crypto_finish_io(struct sr_workunit *);
74 +void		sr_crypto_read(struct cryptop *);
75  void		sr_crypto_calculate_check_hmac_sha1(u_int8_t *, int,
76  		   u_int8_t *, int, u_char *);
77  void		sr_crypto_hotplug(struct sr_discipline *, struct disk *, int);
78 @@ -113,6 +107,7 @@
79  	int i;
80  
81  	/* Fill out discipline members. */
82 +	sd->sd_wu_size = sizeof(struct sr_crypto_wu);
83  	sd->sd_type = SR_MD_CRYPTO;
84  	strlcpy(sd->sd_name, "CRYPTO", sizeof(sd->sd_name));
85  	sd->sd_capabilities = SR_CAP_SYSTEM_DISK | SR_CAP_AUTO_ASSEMBLE;
86 @@ -143,8 +138,14 @@
87  		sr_error(sd->sd_sc, "%s requires exactly one chunk",
88  		    sd->sd_name);
89  		goto done;
90 -        }
91 +	}
92  
93 +	if (coerced_size > SR_CRYPTO_MAXSIZE) {
94 +		sr_error(sd->sd_sc, "%s exceeds maximum size (%lli > %llu)",
95 +		    sd->sd_name, coerced_size, SR_CRYPTO_MAXSIZE);
96 +		goto done;
97 +	}
98 +
99  	/* Create crypto optional metadata. */
100  	omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF,
101  	    M_WAITOK | M_ZERO);
102 @@ -208,7 +209,7 @@
103  
104  	if (data != NULL) {
105  		/* Kernel already has mask key. */
106 -		bcopy(data, sd->mds.mdd_crypto.scr_maskkey,
107 +		memcpy(sd->mds.mdd_crypto.scr_maskkey, data,
108  		    sizeof(sd->mds.mdd_crypto.scr_maskkey));
109  	} else if (bc->bc_key_disk != NODEV) {
110  		/* Read the mask key from the key disk. */
111 @@ -248,117 +249,69 @@
112  }
113  
114  struct sr_crypto_wu *
115 -sr_crypto_wu_get(struct sr_workunit *wu, int encrypt)
116 +sr_crypto_prepare(struct sr_workunit *wu, int encrypt)
117  {
118  	struct scsi_xfer	*xs = wu->swu_xs;
119  	struct sr_discipline	*sd = wu->swu_dis;
120  	struct sr_crypto_wu	*crwu;
121  	struct cryptodesc	*crd;
122  	int			flags, i, n;
123 -	daddr64_t		blk = 0;
124 +	daddr_t			blkno;
125  	u_int			keyndx;
126  
127 -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_get wu: %p encrypt: %d\n",
128 +	DNPRINTF(SR_D_DIS, "%s: sr_crypto_prepare wu %p encrypt %d\n",
129  	    DEVNAME(sd->sd_sc), wu, encrypt);
130  
131 -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
132 -	if ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL)
133 -		TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
134 -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
135 -	if (crwu == NULL)
136 -		panic("sr_crypto_wu_get: out of wus");
137 -
138 +	crwu = (struct sr_crypto_wu *)wu;
139  	crwu->cr_uio.uio_iovcnt = 1;
140  	crwu->cr_uio.uio_iov->iov_len = xs->datalen;
141  	if (xs->flags & SCSI_DATA_OUT) {
142  		crwu->cr_uio.uio_iov->iov_base = crwu->cr_dmabuf;
143 -		bcopy(xs->data, crwu->cr_uio.uio_iov->iov_base, xs->datalen);
144 +		memcpy(crwu->cr_uio.uio_iov->iov_base, xs->data, xs->datalen);
145  	} else
146  		crwu->cr_uio.uio_iov->iov_base = xs->data;
147  
148 -	if (xs->cmdlen == 10)
149 -		blk = _4btol(((struct scsi_rw_big *)xs->cmd)->addr);
150 -	else if (xs->cmdlen == 16)
151 -		blk = _8btol(((struct scsi_rw_16 *)xs->cmd)->addr);
152 -	else if (xs->cmdlen == 6)
153 -		blk = _3btol(((struct scsi_rw *)xs->cmd)->addr);
154 -
155 +	blkno = wu->swu_blk_start;
156  	n = xs->datalen >> DEV_BSHIFT;
157  
158  	/*
159  	 * We preallocated enough crypto descs for up to MAXPHYS of I/O.
160 -	 * Since there may be less than that we need to tweak the linked list
161 +	 * Since there may be less than that we need to tweak the amount
162  	 * of crypto desc structures to be just long enough for our needs.
163  	 */
164 -	crd = crwu->cr_descs;
165 -	for (i = 0; i < ((MAXPHYS >> DEV_BSHIFT) - n); i++) {
166 -		crd = crd->crd_next;
167 -		KASSERT(crd);
168 -	}
169 -	crwu->cr_crp->crp_desc = crd;
170 +	KASSERT(crwu->cr_crp->crp_ndescalloc >= n);
171 +	crwu->cr_crp->crp_ndesc = n;
172  	flags = (encrypt ? CRD_F_ENCRYPT : 0) |
173  	    CRD_F_IV_PRESENT | CRD_F_IV_EXPLICIT;
174  
175 -	/* Select crypto session based on block number */
176 -	keyndx = blk >> SR_CRYPTO_KEY_BLKSHIFT;
177 -	if (keyndx >= SR_CRYPTO_MAXKEYS)
178 -		goto unwind;
179 +	/*
180 +	 * Select crypto session based on block number.
181 +	 *
182 +	 * XXX - this does not handle the case where the read/write spans
183 +	 * across a different key blocks (e.g. 0.5TB boundary). Currently
184 +	 * this is already broken by the use of scr_key[0] below.
185 +	 */
186 +	keyndx = blkno >> SR_CRYPTO_KEY_BLKSHIFT;
187  	crwu->cr_crp->crp_sid = sd->mds.mdd_crypto.scr_sid[keyndx];
188 -	if (crwu->cr_crp->crp_sid == (u_int64_t)-1)
189 -		goto unwind;
190  
191 +	crwu->cr_crp->crp_opaque = crwu;
192  	crwu->cr_crp->crp_ilen = xs->datalen;
193  	crwu->cr_crp->crp_alloctype = M_DEVBUF;
194 +	crwu->cr_crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_NOQUEUE;
195  	crwu->cr_crp->crp_buf = &crwu->cr_uio;
196 -	for (i = 0, crd = crwu->cr_crp->crp_desc; crd;
197 -	    i++, blk++, crd = crd->crd_next) {
198 +	for (i = 0; i < crwu->cr_crp->crp_ndesc; i++, blkno++) {
199 +		crd = &crwu->cr_crp->crp_desc[i];
200  		crd->crd_skip = i << DEV_BSHIFT;
201  		crd->crd_len = DEV_BSIZE;
202  		crd->crd_inject = 0;
203  		crd->crd_flags = flags;
204 -		crd->crd_alg = CRYPTO_AES_XTS;
205 -
206 -		switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
207 -		case SR_CRYPTOA_AES_XTS_128:
208 -			crd->crd_klen = 256;
209 -			break;
210 -		case SR_CRYPTOA_AES_XTS_256:
211 -			crd->crd_klen = 512;
212 -			break;
213 -		default:
214 -			goto unwind;
215 -		}
216 +		crd->crd_alg = sd->mds.mdd_crypto.scr_alg;
217 +		crd->crd_klen = sd->mds.mdd_crypto.scr_klen;
218  		crd->crd_key = sd->mds.mdd_crypto.scr_key[0];
219 -		bcopy(&blk, crd->crd_iv, sizeof(blk));
220 +		memcpy(crd->crd_iv, &blkno, sizeof(blkno));
221  	}
222 -	crwu->cr_wu = wu;
223 -	crwu->cr_crp->crp_opaque = crwu;
224  
225  	return (crwu);
226 -
227 -unwind:
228 -	/* steal the descriptors back from the cryptop */
229 -	crwu->cr_crp->crp_desc = NULL;
230 -
231 -	return (NULL);
232 -}
233 -
234 -void
235 -sr_crypto_wu_put(struct sr_crypto_wu *crwu)
236 -{
237 -	struct cryptop		*crp = crwu->cr_crp;
238 -	struct sr_workunit	*wu = crwu->cr_wu;
239 -	struct sr_discipline	*sd = wu->swu_dis;
240 -
241 -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_put crwu: %p\n",
242 -	    DEVNAME(wu->swu_dis->sd_sc), crwu);
243 -
244 -	/* steal the descriptors back from the cryptop */
245 -	crp->crp_desc = NULL;
246 -
247 -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
248 -	TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
249 -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
250  }
251  
252  int
253 @@ -386,9 +339,8 @@
254  		if (sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint) <
255  		    kdfinfo->genkdf.len)
256  			goto out;
257 -		bcopy(&kdfinfo->genkdf,
258 -		    sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
259 -		    kdfinfo->genkdf.len);
260 +		memcpy(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
261 +		    &kdfinfo->genkdf, kdfinfo->genkdf.len);
262  	}
263  
264  	/* copy mask key to run-time meta data */
265 @@ -396,7 +348,7 @@
266  		if (sizeof(sd->mds.mdd_crypto.scr_maskkey) <
267  		    sizeof(kdfinfo->maskkey))
268  			goto out;
269 -		bcopy(&kdfinfo->maskkey, sd->mds.mdd_crypto.scr_maskkey,
270 +		memcpy(sd->mds.mdd_crypto.scr_maskkey, &kdfinfo->maskkey,
271  		    sizeof(kdfinfo->maskkey));
272  	}
273  
274 @@ -404,7 +356,7 @@
275  	rv = 0;
276  out:
277  	explicit_bzero(kdfinfo, bc->bc_opaque_size);
278 -	free(kdfinfo, M_DEVBUF);
279 +	free(kdfinfo, M_DEVBUF, bc->bc_opaque_size);
280  
281  	return (rv);
282  }
283 @@ -424,7 +376,7 @@
284  		rv = 0;
285  		break;
286  	default:
287 -		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n",
288 +		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %d\n",
289  		    "softraid", alg);
290  		rv = -1;
291  		goto out;
292 @@ -450,7 +402,7 @@
293  		rv = 0;
294  		break;
295  	default:
296 -		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n",
297 +		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %d\n",
298  		    "softraid", alg);
299  		rv = -1;
300  		goto out;
301 @@ -615,6 +567,17 @@
302  		sr_error(sd->sd_sc, "incorrect key or passphrase");
303  		rv = EPERM;
304  		goto out;
305 +	}
306 +
307 +	/* Copy new KDF hint to metadata, if supplied. */
308 +	if (kdfinfo2->flags & SR_CRYPTOKDF_HINT) {
309 +		if (kdfinfo2->genkdf.len >
310 +		    sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint))
311 +			goto out;
312 +		explicit_bzero(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
313 +		    sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint));
314 +		memcpy(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
315 +		    &kdfinfo2->genkdf, kdfinfo2->genkdf.len);
316  	}
317  
318  	/* Mask the disk keys. */
319 @@ -630,7 +593,7 @@
320  	    sizeof(sd->mds.mdd_crypto.scr_key), check_digest);
321  
322  	/* Copy new encrypted key and HMAC to metadata. */
323 -	bcopy(check_digest, sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac,
324 +	memcpy(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac, check_digest,
325  	    sizeof(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac));
326  
327  	rv = 0; /* Success */
328 @@ -638,7 +601,7 @@
329  out:
330  	if (p) {
331  		explicit_bzero(p, ksz);
332 -		free(p, M_DEVBUF);
333 +		free(p, M_DEVBUF, ksz);
334  	}
335  
336  	explicit_bzero(check_digest, sizeof(check_digest));
337 @@ -686,7 +649,7 @@
338  		DNPRINTF(SR_D_META,"%s: sr_crypto_create_key_disk cannot "
339  		    "open %s\n", DEVNAME(sc), devname);
340  		vput(vn);
341 -		goto fail;
342 +		goto done;
343  	}
344  	open = 1; /* close dev on error */
345  
346 @@ -696,19 +659,12 @@
347  	    FREAD, NOCRED, curproc)) {
348  		DNPRINTF(SR_D_META, "%s: sr_crypto_create_key_disk ioctl "
349  		    "failed\n", DEVNAME(sc));
350 -		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
351 -		vput(vn);
352 -		goto fail;
353 +		goto done;
354  	}
355 -	if (label.d_secsize != DEV_BSIZE) {
356 -		sr_error(sc, "%s has unsupported sector size (%d)",
357 -		    devname, label.d_secsize);
358 -		goto fail;
359 -	}
360  	if (label.d_partitions[part].p_fstype != FS_RAID) {
361 -		sr_error(sc, "%s partition not of type RAID (%d)\n",
362 +		sr_error(sc, "%s partition not of type RAID (%d)",
363  		    devname, label.d_partitions[part].p_fstype);
364 -		goto fail;
365 +		goto done;
366  	}
367  
368  	/*
369 @@ -728,7 +684,7 @@
370  	km->scmi.scm_size = 0;
371  	km->scmi.scm_coerced_size = 0;
372  	strlcpy(km->scmi.scm_devname, devname, sizeof(km->scmi.scm_devname));
373 -	bcopy(&sd->sd_meta->ssdi.ssd_uuid, &km->scmi.scm_uuid,
374 +	memcpy(&km->scmi.scm_uuid, &sd->sd_meta->ssdi.ssd_uuid,
375  	    sizeof(struct sr_uuid));
376  
377  	sr_checksum(sc, km, &km->scm_checksum,
378 @@ -745,7 +701,7 @@
379  	sm->ssdi.ssd_version = SR_META_VERSION;
380  	sm->ssd_ondisk = 0;
381  	sm->ssdi.ssd_vol_flags = 0;
382 -	bcopy(&sd->sd_meta->ssdi.ssd_uuid, &sm->ssdi.ssd_uuid,
383 +	memcpy(&sm->ssdi.ssd_uuid, &sd->sd_meta->ssdi.ssd_uuid,
384  	    sizeof(struct sr_uuid));
385  	sm->ssdi.ssd_chunk_no = 1;
386  	sm->ssdi.ssd_volid = SR_KEYDISK_VOLID;
387 @@ -785,7 +741,7 @@
388  	omi->omi_som->som_type = SR_OPT_KEYDISK;
389  	omi->omi_som->som_length = sizeof(struct sr_meta_keydisk);
390  	skm = (struct sr_meta_keydisk *)omi->omi_som;
391 -	bcopy(sd->mds.mdd_crypto.scr_maskkey, &skm->skm_maskkey,
392 +	memcpy(&skm->skm_maskkey, sd->mds.mdd_crypto.scr_maskkey,
393  	    sizeof(skm->skm_maskkey));
394  	SLIST_INSERT_HEAD(&fakesd->sd_meta_opt, omi, omi_link);
395  	fakesd->sd_meta->ssdi.ssd_opt_no++;
396 @@ -799,19 +755,16 @@
397  	goto done;
398  
399  fail:
400 -	if (key_disk)
401 -		free(key_disk, M_DEVBUF);
402 +	free(key_disk, M_DEVBUF, sizeof(struct sr_chunk));
403  	key_disk = NULL;
404  
405  done:
406 -	if (omi)
407 -		free(omi, M_DEVBUF);
408 +	free(omi, M_DEVBUF, sizeof(struct sr_meta_opt_item));
409  	if (fakesd && fakesd->sd_vol.sv_chunks)
410 -		free(fakesd->sd_vol.sv_chunks, M_DEVBUF);
411 -	if (fakesd)
412 -		free(fakesd, M_DEVBUF);
413 -	if (sm)
414 -		free(sm, M_DEVBUF);
415 +		free(fakesd->sd_vol.sv_chunks, M_DEVBUF,
416 +		    sizeof(struct sr_chunk *));
417 +	free(fakesd, M_DEVBUF, sizeof(struct sr_discipline));
418 +	free(sm, M_DEVBUF, sizeof(struct sr_metadata));
419  	if (open) {
420  		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
421  		vput(vn);
422 @@ -855,7 +808,7 @@
423  		sr_error(sc, "cannot open key disk %s", devname);
424  		goto done;
425  	}
426 -	if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) {
427 +	if (VOP_OPEN(vn, FREAD, NOCRED, curproc)) {
428  		DNPRINTF(SR_D_META,"%s: sr_crypto_read_key_disk cannot "
429  		    "open %s\n", DEVNAME(sc), devname);
430  		vput(vn);
431 @@ -869,17 +822,10 @@
432  	    NOCRED, curproc)) {
433  		DNPRINTF(SR_D_META, "%s: sr_crypto_read_key_disk ioctl "
434  		    "failed\n", DEVNAME(sc));
435 -		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
436 -		vput(vn);
437  		goto done;
438  	}
439 -	if (label.d_secsize != DEV_BSIZE) {
440 -		sr_error(sc, "%s has unsupported sector size (%d)",
441 -		    devname, label.d_secsize);
442 -		goto done;
443 -	}
444  	if (label.d_partitions[part].p_fstype != FS_RAID) {
445 -		sr_error(sc, "%s partition not of type RAID (%d)\n",
446 +		sr_error(sc, "%s partition not of type RAID (%d)",
447  		    devname, label.d_partitions[part].p_fstype);
448  		goto done;
449  	}
450 @@ -887,7 +833,7 @@
451  	/*
452  	 * Read and validate key disk metadata.
453  	 */
454 -	sm = malloc(SR_META_SIZE * 512, M_DEVBUF, M_WAITOK | M_ZERO);
455 +	sm = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_WAITOK | M_ZERO);
456  	if (sr_meta_native_read(sd, dev, sm, NULL)) {
457  		sr_error(sc, "native bootprobe could not read native metadata");
458  		goto done;
459 @@ -911,7 +857,7 @@
460  	key_disk->src_vn = vn;
461  	key_disk->src_size = 0;
462  
463 -	bcopy((struct sr_meta_chunk *)(sm + 1), &key_disk->src_meta,
464 +	memcpy(&key_disk->src_meta, (struct sr_meta_chunk *)(sm + 1),
465  	    sizeof(key_disk->src_meta));
466  
467  	/* Read mask key from optional metadata. */
468 @@ -920,13 +866,12 @@
469  		omh = omi->omi_som;
470  		if (omh->som_type == SR_OPT_KEYDISK) {
471  			skm = (struct sr_meta_keydisk *)omh;
472 -			bcopy(&skm->skm_maskkey,
473 -			    sd->mds.mdd_crypto.scr_maskkey,
474 +			memcpy(sd->mds.mdd_crypto.scr_maskkey, &skm->skm_maskkey,
475  			    sizeof(sd->mds.mdd_crypto.scr_maskkey));
476  		} else if (omh->som_type == SR_OPT_CRYPTO) {
477  			/* Original keydisk format with key in crypto area. */
478 -			bcopy(omh + sizeof(struct sr_meta_opt_hdr),
479 -			    sd->mds.mdd_crypto.scr_maskkey,
480 +			memcpy(sd->mds.mdd_crypto.scr_maskkey,
481 +			    omh + sizeof(struct sr_meta_opt_hdr),
482  			    sizeof(sd->mds.mdd_crypto.scr_maskkey));
483  		}
484  	}
485 @@ -934,15 +879,13 @@
486  	open = 0;
487  
488  done:
489 -	for (omi = SLIST_FIRST(&som); omi != SLIST_END(&som); omi = omi_next) {
490 +	for (omi = SLIST_FIRST(&som); omi != NULL; omi = omi_next) {
491  		omi_next = SLIST_NEXT(omi, omi_link);
492 -		if (omi->omi_som)
493 -			free(omi->omi_som, M_DEVBUF);
494 -		free(omi, M_DEVBUF);
495 +		free(omi->omi_som, M_DEVBUF, 0);
496 +		free(omi, M_DEVBUF, sizeof(struct sr_meta_opt_item));
497  	}
498  
499 -	if (sm)
500 -		free(sm, M_DEVBUF);
501 +	free(sm, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
502  
503  	if (vn && open) {
504  		VOP_CLOSE(vn, FREAD, NOCRED, curproc);
505 @@ -950,18 +893,45 @@
506  	}
507  
508  	return key_disk;
509 +}
510 +
511 +static void
512 +sr_crypto_free_sessions(struct sr_discipline *sd)
513 +{
514 +	u_int			i;
515 +
516 +	for (i = 0; i < SR_CRYPTO_MAXKEYS; i++) {
517 +		if (sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1) {
518 +			crypto_freesession(sd->mds.mdd_crypto.scr_sid[i]);
519 +			sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
520 +		}
521 +	}
522  }
523  
524  int
525  sr_crypto_alloc_resources(struct sr_discipline *sd)
526  {
527 -	struct cryptoini	cri;
528 +	struct sr_workunit	*wu;
529  	struct sr_crypto_wu	*crwu;
530 +	struct cryptoini	cri;
531  	u_int			num_keys, i;
532  
533  	DNPRINTF(SR_D_DIS, "%s: sr_crypto_alloc_resources\n",
534  	    DEVNAME(sd->sd_sc));
535  
536 +	sd->mds.mdd_crypto.scr_alg = CRYPTO_AES_XTS;
537 +	switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
538 +	case SR_CRYPTOA_AES_XTS_128:
539 +		sd->mds.mdd_crypto.scr_klen = 256;
540 +		break;
541 +	case SR_CRYPTOA_AES_XTS_256:
542 +		sd->mds.mdd_crypto.scr_klen = 512;
543 +		break;
544 +	default:
545 +		sr_error(sd->sd_sc, "unknown crypto algorithm");
546 +		return (EINVAL);
547 +	}
548 +
549  	for (i = 0; i < SR_CRYPTO_MAXKEYS; i++)
550  		sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
551  
552 @@ -979,61 +949,34 @@
553  	}
554  
555  	/*
556 -	 * For each wu allocate the uio, iovec and crypto structures.
557 -	 * these have to be allocated now because during runtime we can't
558 -	 * fail an allocation without failing the io (which can cause real
559 +	 * For each work unit allocate the uio, iovec and crypto structures.
560 +	 * These have to be allocated now because during runtime we cannot
561 +	 * fail an allocation without failing the I/O (which can cause real
562  	 * problems).
563  	 */
564 -	mtx_init(&sd->mds.mdd_crypto.scr_mutex, IPL_BIO);
565 -	TAILQ_INIT(&sd->mds.mdd_crypto.scr_wus);
566 -	for (i = 0; i < sd->sd_max_wu; i++) {
567 -		crwu = malloc(sizeof(*crwu), M_DEVBUF,
568 -		    M_WAITOK | M_ZERO | M_CANFAIL);
569 -		if (crwu == NULL)
570 -		    return (ENOMEM);
571 -		/* put it on the list now so if we fail it'll be freed */
572 -		mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
573 -		TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
574 -		mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
575 -
576 +	TAILQ_FOREACH(wu, &sd->sd_wu, swu_next) {
577 +		crwu = (struct sr_crypto_wu *)wu;
578  		crwu->cr_uio.uio_iov = &crwu->cr_iov;
579  		crwu->cr_dmabuf = dma_alloc(MAXPHYS, PR_WAITOK);
580  		crwu->cr_crp = crypto_getreq(MAXPHYS >> DEV_BSHIFT);
581  		if (crwu->cr_crp == NULL)
582  			return (ENOMEM);
583 -		/* steal the list of cryptodescs */
584 -		crwu->cr_descs = crwu->cr_crp->crp_desc;
585 -		crwu->cr_crp->crp_desc = NULL;
586  	}
587  
588 -	bzero(&cri, sizeof(cri));
589 -	cri.cri_alg = CRYPTO_AES_XTS;
590 -	switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
591 -	case SR_CRYPTOA_AES_XTS_128:
592 -		cri.cri_klen = 256;
593 -		break;
594 -	case SR_CRYPTOA_AES_XTS_256:
595 -		cri.cri_klen = 512;
596 -		break;
597 -	default:
598 -		return (EINVAL);
599 -	}
600 +	memset(&cri, 0, sizeof(cri));
601 +	cri.cri_alg = sd->mds.mdd_crypto.scr_alg;
602 +	cri.cri_klen = sd->mds.mdd_crypto.scr_klen;
603  
604 -	/* Allocate a session for every 2^SR_CRYPTO_KEY_BLKSHIFT blocks */
605 -	num_keys = sd->sd_meta->ssdi.ssd_size >> SR_CRYPTO_KEY_BLKSHIFT;
606 -	if (num_keys >= SR_CRYPTO_MAXKEYS)
607 +	/* Allocate a session for every 2^SR_CRYPTO_KEY_BLKSHIFT blocks. */
608 +	num_keys = ((sd->sd_meta->ssdi.ssd_size - 1) >>
609 +	    SR_CRYPTO_KEY_BLKSHIFT) + 1;
610 +	if (num_keys > SR_CRYPTO_MAXKEYS)
611  		return (EFBIG);
612 -	for (i = 0; i <= num_keys; i++) {
613 +	for (i = 0; i < num_keys; i++) {
614  		cri.cri_key = sd->mds.mdd_crypto.scr_key[i];
615  		if (crypto_newsession(&sd->mds.mdd_crypto.scr_sid[i],
616  		    &cri, 0) != 0) {
617 -			for (i = 0;
618 -			     sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1;
619 -			     i++) {
620 -				crypto_freesession(
621 -				    sd->mds.mdd_crypto.scr_sid[i]);
622 -				sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
623 -			}
624 +			sr_crypto_free_sessions(sd);
625  			return (EINVAL);
626  		}
627  	}
628 @@ -1046,39 +989,30 @@
629  void
630  sr_crypto_free_resources(struct sr_discipline *sd)
631  {
632 +	struct sr_workunit	*wu;
633  	struct sr_crypto_wu	*crwu;
634 -	u_int			i;
635  
636  	DNPRINTF(SR_D_DIS, "%s: sr_crypto_free_resources\n",
637  	    DEVNAME(sd->sd_sc));
638  
639  	if (sd->mds.mdd_crypto.key_disk != NULL) {
640 -		explicit_bzero(sd->mds.mdd_crypto.key_disk, sizeof
641 -		    sd->mds.mdd_crypto.key_disk);
642 -		free(sd->mds.mdd_crypto.key_disk, M_DEVBUF);
643 +		explicit_bzero(sd->mds.mdd_crypto.key_disk,
644 +		    sizeof(*sd->mds.mdd_crypto.key_disk));
645 +		free(sd->mds.mdd_crypto.key_disk, M_DEVBUF,
646 +		    sizeof(*sd->mds.mdd_crypto.key_disk));
647  	}
648  
649  	sr_hotplug_unregister(sd, sr_crypto_hotplug);
650  
651 -	for (i = 0; sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1; i++) {
652 -		crypto_freesession(sd->mds.mdd_crypto.scr_sid[i]);
653 -		sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
654 -	}
655 +	sr_crypto_free_sessions(sd);
656  
657 -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
658 -	while ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL) {
659 -		TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
660 -
661 -		if (crwu->cr_dmabuf != NULL)
662 +	TAILQ_FOREACH(wu, &sd->sd_wu, swu_next) {
663 +		crwu = (struct sr_crypto_wu *)wu;
664 +		if (crwu->cr_dmabuf)
665  			dma_free(crwu->cr_dmabuf, MAXPHYS);
666 -		if (crwu->cr_crp) {
667 -			/* twiddle cryptoreq back */
668 -			crwu->cr_crp->crp_desc = crwu->cr_descs;
669 +		if (crwu->cr_crp)
670  			crypto_freereq(crwu->cr_crp);
671 -		}
672 -		free(crwu, M_DEVBUF);
673  	}
674 -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
675  
676  	sr_wu_free(sd);
677  	sr_ccb_free(sd);
678 @@ -1165,65 +1099,60 @@
679  sr_crypto_rw(struct sr_workunit *wu)
680  {
681  	struct sr_crypto_wu	*crwu;
682 -	int			s, rv = 0;
683 +	daddr_t			blkno;
684 +	int			rv = 0;
685  
686 -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu: %p\n",
687 +	DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu %p\n",
688  	    DEVNAME(wu->swu_dis->sd_sc), wu);
689  
690 -	if (wu->swu_xs->flags & SCSI_DATA_OUT) {
691 -		crwu = sr_crypto_wu_get(wu, 1);
692 -		if (crwu == NULL)
693 -			return (1);
694 +	if (sr_validate_io(wu, &blkno, "sr_crypto_rw"))
695 +		return (1);
696 +
697 +	if (wu->swu_xs->flags & SCSI_DATA_OUT) {
698 +		crwu = sr_crypto_prepare(wu, 1);
699  		crwu->cr_crp->crp_callback = sr_crypto_write;
700 -		s = splvm();
701 -		if (crypto_invoke(crwu->cr_crp))
702 -			rv = 1;
703 -		else
704 +		rv = crypto_dispatch(crwu->cr_crp);
705 +		if (rv == 0)
706  			rv = crwu->cr_crp->crp_etype;
707 -		splx(s);
708  	} else
709 -		rv = sr_crypto_rw2(wu, NULL);
710 +		rv = sr_crypto_dev_rw(wu, NULL);
711  
712  	return (rv);
713  }
714  
715 -int
716 +void
717  sr_crypto_write(struct cryptop *crp)
718  {
719  	struct sr_crypto_wu	*crwu = crp->crp_opaque;
720 -	struct sr_workunit	*wu = crwu->cr_wu;
721 +	struct sr_workunit	*wu = &crwu->cr_wu;
722  	int			s;
723  
724 -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %x xs: %x\n",
725 +	DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %p xs: %p\n",
726  	    DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
727  
728  	if (crp->crp_etype) {
729  		/* fail io */
730  		wu->swu_xs->error = XS_DRIVER_STUFFUP;
731  		s = splbio();
732 -		sr_crypto_finish_io(wu);
733 +		sr_scsi_done(wu->swu_dis, wu->swu_xs);
734  		splx(s);
735  	}
736  
737 -	return (sr_crypto_rw2(wu, crwu));
738 +	sr_crypto_dev_rw(wu, crwu);
739  }
740  
741  int
742 -sr_crypto_rw2(struct sr_workunit *wu, struct sr_crypto_wu *crwu)
743 +sr_crypto_dev_rw(struct sr_workunit *wu, struct sr_crypto_wu *crwu)
744  {
745  	struct sr_discipline	*sd = wu->swu_dis;
746  	struct scsi_xfer	*xs = wu->swu_xs;
747  	struct sr_ccb		*ccb;
748  	struct uio		*uio;
749 -	int			s;
750 -	daddr64_t		blk;
751 +	daddr_t			blkno;
752  
753 -	if (sr_validate_io(wu, &blk, "sr_crypto_rw2"))
754 -		goto bad;
755 +	blkno = wu->swu_blk_start;
756  
757 -	blk += sd->sd_meta->ssd_data_offset;
758 -
759 -	ccb = sr_ccb_rw(sd, 0, blk, xs->datalen, xs->data, xs->flags, 0);
760 +	ccb = sr_ccb_rw(sd, 0, blkno, xs->datalen, xs->data, xs->flags, 0);
761  	if (!ccb) {
762  		/* should never happen but handle more gracefully */
763  		printf("%s: %s: too many ccbs queued\n",
764 @@ -1236,17 +1165,10 @@
765  		ccb->ccb_opaque = crwu;
766  	}
767  	sr_wu_enqueue_ccb(wu, ccb);
768 +	sr_schedule_wu(wu);
769  
770 -	s = splbio();
771 -
772 -	if (sr_check_io_collision(wu))
773 -		goto queued;
774 -
775 -	sr_raid_startwu(wu);
776 -
777 -queued:
778 -	splx(s);
779  	return (0);
780 +
781  bad:
782  	/* wu is unwound by sr_wu_put */
783  	if (crwu)
784 @@ -1259,77 +1181,39 @@
785  {
786  	struct scsi_xfer	*xs = wu->swu_xs;
787  	struct sr_crypto_wu	*crwu;
788 -	struct sr_ccb		*ccb;
789  	int			s;
790  
791  	/* If this was a successful read, initiate decryption of the data. */
792  	if (ISSET(xs->flags, SCSI_DATA_IN) && xs->error == XS_NOERROR) {
793 -		/* only fails on implementation error */
794 -		crwu = sr_crypto_wu_get(wu, 0);
795 -		if (crwu == NULL)
796 -			panic("sr_crypto_intr: no wu");
797 +		crwu = sr_crypto_prepare(wu, 0);
798  		crwu->cr_crp->crp_callback = sr_crypto_read;
799 -		ccb = TAILQ_FIRST(&wu->swu_ccb);
800 -		if (ccb == NULL)
801 -			panic("sr_crypto_done: no ccbs on workunit");
802 -		ccb->ccb_opaque = crwu;
803 -		DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: crypto_invoke %p\n",
804 +		DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_dispatch %p\n",
805  		    DEVNAME(wu->swu_dis->sd_sc), crwu->cr_crp);
806 -		s = splvm();
807 -		crypto_invoke(crwu->cr_crp);
808 -		splx(s);
809 +		crypto_dispatch(crwu->cr_crp);
810  		return;
811  	}
812  
813  	s = splbio();
814 -	sr_crypto_finish_io(wu);
815 +	sr_scsi_done(wu->swu_dis, wu->swu_xs);
816  	splx(s);
817  }
818  
819  void
820 -sr_crypto_finish_io(struct sr_workunit *wu)
821 -{
822 -	struct sr_discipline	*sd = wu->swu_dis;
823 -	struct scsi_xfer	*xs = wu->swu_xs;
824 -	struct sr_ccb		*ccb;
825 -#ifdef SR_DEBUG
826 -	struct sr_softc		*sc = sd->sd_sc;
827 -#endif /* SR_DEBUG */
828 -
829 -	splassert(IPL_BIO);
830 -
831 -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_finish_io: wu %x xs: %x\n",
832 -	    DEVNAME(sc), wu, xs);
833 -
834 -	if (wu->swu_cb_active == 1)
835 -		panic("%s: sr_crypto_finish_io", DEVNAME(sd->sd_sc));
836 -	TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) {
837 -		if (ccb->ccb_opaque == NULL)
838 -			continue;
839 -		sr_crypto_wu_put(ccb->ccb_opaque);
840 -	}
841 -
842 -	sr_scsi_done(sd, xs);
843 -}
844 -
845 -int
846  sr_crypto_read(struct cryptop *crp)
847  {
848  	struct sr_crypto_wu	*crwu = crp->crp_opaque;
849 -	struct sr_workunit	*wu = crwu->cr_wu;
850 +	struct sr_workunit	*wu = &crwu->cr_wu;
851  	int			s;
852  
853 -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %x xs: %x\n",
854 +	DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %p xs: %p\n",
855  	    DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
856  
857  	if (crp->crp_etype)
858  		wu->swu_xs->error = XS_DRIVER_STUFFUP;
859  
860  	s = splbio();
861 -	sr_crypto_finish_io(wu);
862 +	sr_scsi_done(wu->swu_dis, wu->swu_xs);
863  	splx(s);
864 -
865 -	return (0);
866  }
867  
868  void