1 c285a1f8 2020-11-06 stsp --- test021.left.txt
  2 c285a1f8 2020-11-06 stsp +++ test021.right.txt
  3 c285a1f8 2020-11-06 stsp @@ -1,4 +1,4 @@
  4 c285a1f8 2020-11-06 stsp -/* $OpenBSD: softraid_crypto.c,v 1.91 2013/03/31 15:44:52 jsing Exp $ */
  5 c285a1f8 2020-11-06 stsp +/* $OpenBSD: softraid_crypto.c,v 1.139 2020/07/13 00:06:22 kn Exp $ */
  6 c285a1f8 2020-11-06 stsp  /*
  7 c285a1f8 2020-11-06 stsp   * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  8 c285a1f8 2020-11-06 stsp   * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
  9 c285a1f8 2020-11-06 stsp @@ -25,7 +25,6 @@
 10 c285a1f8 2020-11-06 stsp  #include <sys/buf.h>
 11 c285a1f8 2020-11-06 stsp  #include <sys/device.h>
 12 c285a1f8 2020-11-06 stsp  #include <sys/ioctl.h>
 13 c285a1f8 2020-11-06 stsp -#include <sys/proc.h>
 14 c285a1f8 2020-11-06 stsp  #include <sys/malloc.h>
 15 c285a1f8 2020-11-06 stsp  #include <sys/pool.h>
 16 c285a1f8 2020-11-06 stsp  #include <sys/kernel.h>
 17 c285a1f8 2020-11-06 stsp @@ -34,6 +33,7 @@
 18 c285a1f8 2020-11-06 stsp  #include <sys/queue.h>
 19 c285a1f8 2020-11-06 stsp  #include <sys/fcntl.h>
 20 c285a1f8 2020-11-06 stsp  #include <sys/disklabel.h>
 21 c285a1f8 2020-11-06 stsp +#include <sys/vnode.h>
 22 c285a1f8 2020-11-06 stsp  #include <sys/mount.h>
 23 c285a1f8 2020-11-06 stsp  #include <sys/sensors.h>
 24 c285a1f8 2020-11-06 stsp  #include <sys/stat.h>
 25 c285a1f8 2020-11-06 stsp @@ -42,7 +42,6 @@
 26 c285a1f8 2020-11-06 stsp  #include <sys/dkio.h>
 27 c285a1f8 2020-11-06 stsp  
 28 c285a1f8 2020-11-06 stsp  #include <crypto/cryptodev.h>
 29 c285a1f8 2020-11-06 stsp -#include <crypto/cryptosoft.h>
 30 c285a1f8 2020-11-06 stsp  #include <crypto/rijndael.h>
 31 c285a1f8 2020-11-06 stsp  #include <crypto/md5.h>
 32 c285a1f8 2020-11-06 stsp  #include <crypto/sha1.h>
 33 c285a1f8 2020-11-06 stsp @@ -54,7 +53,6 @@
 34 c285a1f8 2020-11-06 stsp  #include <scsi/scsi_disk.h>
 35 c285a1f8 2020-11-06 stsp  
 36 c285a1f8 2020-11-06 stsp  #include <dev/softraidvar.h>
 37 c285a1f8 2020-11-06 stsp -#include <dev/rndvar.h>
 38 c285a1f8 2020-11-06 stsp  
 39 c285a1f8 2020-11-06 stsp  /*
 40 c285a1f8 2020-11-06 stsp   * The per-I/O data that we need to preallocate. We cannot afford to allow I/O
 41 c285a1f8 2020-11-06 stsp @@ -62,18 +60,15 @@
 42 c285a1f8 2020-11-06 stsp   * because we assert that only one ccb per WU will ever be active.
 43 c285a1f8 2020-11-06 stsp   */
 44 c285a1f8 2020-11-06 stsp  struct sr_crypto_wu {
 45 c285a1f8 2020-11-06 stsp -	TAILQ_ENTRY(sr_crypto_wu)	 cr_link;
 46 c285a1f8 2020-11-06 stsp +	struct sr_workunit		 cr_wu;		/* Must be first. */
 47 c285a1f8 2020-11-06 stsp  	struct uio			 cr_uio;
 48 c285a1f8 2020-11-06 stsp  	struct iovec			 cr_iov;
 49 c285a1f8 2020-11-06 stsp  	struct cryptop	 		*cr_crp;
 50 c285a1f8 2020-11-06 stsp -	struct cryptodesc		*cr_descs;
 51 c285a1f8 2020-11-06 stsp -	struct sr_workunit		*cr_wu;
 52 c285a1f8 2020-11-06 stsp  	void				*cr_dmabuf;
 53 c285a1f8 2020-11-06 stsp  };
 54 c285a1f8 2020-11-06 stsp  
 55 c285a1f8 2020-11-06 stsp  
 56 c285a1f8 2020-11-06 stsp -struct sr_crypto_wu *sr_crypto_wu_get(struct sr_workunit *, int);
 57 c285a1f8 2020-11-06 stsp -void		sr_crypto_wu_put(struct sr_crypto_wu *);
 58 c285a1f8 2020-11-06 stsp +struct sr_crypto_wu *sr_crypto_prepare(struct sr_workunit *, int);
 59 c285a1f8 2020-11-06 stsp  int		sr_crypto_create_keys(struct sr_discipline *);
 60 c285a1f8 2020-11-06 stsp  int		sr_crypto_get_kdf(struct bioc_createraid *,
 61 c285a1f8 2020-11-06 stsp  		    struct sr_discipline *);
 62 c285a1f8 2020-11-06 stsp @@ -92,12 +87,11 @@
 63 c285a1f8 2020-11-06 stsp  		    struct bioc_discipline *);
 64 c285a1f8 2020-11-06 stsp  int		sr_crypto_meta_opt_handler(struct sr_discipline *,
 65 c285a1f8 2020-11-06 stsp  		    struct sr_meta_opt_hdr *);
 66 c285a1f8 2020-11-06 stsp -int		sr_crypto_write(struct cryptop *);
 67 c285a1f8 2020-11-06 stsp +void		sr_crypto_write(struct cryptop *);
 68 c285a1f8 2020-11-06 stsp  int		sr_crypto_rw(struct sr_workunit *);
 69 c285a1f8 2020-11-06 stsp -int		sr_crypto_rw2(struct sr_workunit *, struct sr_crypto_wu *);
 70 c285a1f8 2020-11-06 stsp +int		sr_crypto_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
 71 c285a1f8 2020-11-06 stsp  void		sr_crypto_done(struct sr_workunit *);
 72 c285a1f8 2020-11-06 stsp -int		sr_crypto_read(struct cryptop *);
 73 c285a1f8 2020-11-06 stsp -void		sr_crypto_finish_io(struct sr_workunit *);
 74 c285a1f8 2020-11-06 stsp +void		sr_crypto_read(struct cryptop *);
 75 c285a1f8 2020-11-06 stsp  void		sr_crypto_calculate_check_hmac_sha1(u_int8_t *, int,
 76 c285a1f8 2020-11-06 stsp  		   u_int8_t *, int, u_char *);
 77 c285a1f8 2020-11-06 stsp  void		sr_crypto_hotplug(struct sr_discipline *, struct disk *, int);
 78 c285a1f8 2020-11-06 stsp @@ -113,6 +107,7 @@
 79 c285a1f8 2020-11-06 stsp  	int i;
 80 c285a1f8 2020-11-06 stsp  
 81 c285a1f8 2020-11-06 stsp  	/* Fill out discipline members. */
 82 c285a1f8 2020-11-06 stsp +	sd->sd_wu_size = sizeof(struct sr_crypto_wu);
 83 c285a1f8 2020-11-06 stsp  	sd->sd_type = SR_MD_CRYPTO;
 84 c285a1f8 2020-11-06 stsp  	strlcpy(sd->sd_name, "CRYPTO", sizeof(sd->sd_name));
 85 c285a1f8 2020-11-06 stsp  	sd->sd_capabilities = SR_CAP_SYSTEM_DISK | SR_CAP_AUTO_ASSEMBLE;
 86 c285a1f8 2020-11-06 stsp @@ -143,8 +138,14 @@
 87 c285a1f8 2020-11-06 stsp  		sr_error(sd->sd_sc, "%s requires exactly one chunk",
 88 c285a1f8 2020-11-06 stsp  		    sd->sd_name);
 89 c285a1f8 2020-11-06 stsp  		goto done;
 90 c285a1f8 2020-11-06 stsp -        }
 91 c285a1f8 2020-11-06 stsp +	}
 92 c285a1f8 2020-11-06 stsp  
 93 c285a1f8 2020-11-06 stsp +	if (coerced_size > SR_CRYPTO_MAXSIZE) {
 94 c285a1f8 2020-11-06 stsp +		sr_error(sd->sd_sc, "%s exceeds maximum size (%lli > %llu)",
 95 c285a1f8 2020-11-06 stsp +		    sd->sd_name, coerced_size, SR_CRYPTO_MAXSIZE);
 96 c285a1f8 2020-11-06 stsp +		goto done;
 97 c285a1f8 2020-11-06 stsp +	}
 98 c285a1f8 2020-11-06 stsp +
 99 c285a1f8 2020-11-06 stsp  	/* Create crypto optional metadata. */
100 c285a1f8 2020-11-06 stsp  	omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF,
101 c285a1f8 2020-11-06 stsp  	    M_WAITOK | M_ZERO);
102 c285a1f8 2020-11-06 stsp @@ -208,7 +209,7 @@
103 c285a1f8 2020-11-06 stsp  
104 c285a1f8 2020-11-06 stsp  	if (data != NULL) {
105 c285a1f8 2020-11-06 stsp  		/* Kernel already has mask key. */
106 c285a1f8 2020-11-06 stsp -		bcopy(data, sd->mds.mdd_crypto.scr_maskkey,
107 c285a1f8 2020-11-06 stsp +		memcpy(sd->mds.mdd_crypto.scr_maskkey, data,
108 c285a1f8 2020-11-06 stsp  		    sizeof(sd->mds.mdd_crypto.scr_maskkey));
109 c285a1f8 2020-11-06 stsp  	} else if (bc->bc_key_disk != NODEV) {
110 c285a1f8 2020-11-06 stsp  		/* Read the mask key from the key disk. */
111 405e89a6 2022-08-02 op @@ -248,117 +249,69 @@
112 c285a1f8 2020-11-06 stsp  }
113 c285a1f8 2020-11-06 stsp  
114 c285a1f8 2020-11-06 stsp  struct sr_crypto_wu *
115 c285a1f8 2020-11-06 stsp -sr_crypto_wu_get(struct sr_workunit *wu, int encrypt)
116 c285a1f8 2020-11-06 stsp +sr_crypto_prepare(struct sr_workunit *wu, int encrypt)
117 c285a1f8 2020-11-06 stsp  {
118 c285a1f8 2020-11-06 stsp  	struct scsi_xfer	*xs = wu->swu_xs;
119 c285a1f8 2020-11-06 stsp  	struct sr_discipline	*sd = wu->swu_dis;
120 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu;
121 c285a1f8 2020-11-06 stsp  	struct cryptodesc	*crd;
122 c285a1f8 2020-11-06 stsp  	int			flags, i, n;
123 c285a1f8 2020-11-06 stsp -	daddr64_t		blk = 0;
124 c285a1f8 2020-11-06 stsp +	daddr_t			blkno;
125 c285a1f8 2020-11-06 stsp  	u_int			keyndx;
126 c285a1f8 2020-11-06 stsp  
127 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_get wu: %p encrypt: %d\n",
128 c285a1f8 2020-11-06 stsp +	DNPRINTF(SR_D_DIS, "%s: sr_crypto_prepare wu %p encrypt %d\n",
129 c285a1f8 2020-11-06 stsp  	    DEVNAME(sd->sd_sc), wu, encrypt);
130 c285a1f8 2020-11-06 stsp  
131 c285a1f8 2020-11-06 stsp -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
132 c285a1f8 2020-11-06 stsp -	if ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL)
133 c285a1f8 2020-11-06 stsp -		TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
134 c285a1f8 2020-11-06 stsp -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
135 c285a1f8 2020-11-06 stsp -	if (crwu == NULL)
136 c285a1f8 2020-11-06 stsp -		panic("sr_crypto_wu_get: out of wus");
137 c285a1f8 2020-11-06 stsp -
138 c285a1f8 2020-11-06 stsp +	crwu = (struct sr_crypto_wu *)wu;
139 c285a1f8 2020-11-06 stsp  	crwu->cr_uio.uio_iovcnt = 1;
140 c285a1f8 2020-11-06 stsp  	crwu->cr_uio.uio_iov->iov_len = xs->datalen;
141 c285a1f8 2020-11-06 stsp  	if (xs->flags & SCSI_DATA_OUT) {
142 c285a1f8 2020-11-06 stsp  		crwu->cr_uio.uio_iov->iov_base = crwu->cr_dmabuf;
143 c285a1f8 2020-11-06 stsp -		bcopy(xs->data, crwu->cr_uio.uio_iov->iov_base, xs->datalen);
144 c285a1f8 2020-11-06 stsp +		memcpy(crwu->cr_uio.uio_iov->iov_base, xs->data, xs->datalen);
145 c285a1f8 2020-11-06 stsp  	} else
146 c285a1f8 2020-11-06 stsp  		crwu->cr_uio.uio_iov->iov_base = xs->data;
147 c285a1f8 2020-11-06 stsp  
148 c285a1f8 2020-11-06 stsp -	if (xs->cmdlen == 10)
149 c285a1f8 2020-11-06 stsp -		blk = _4btol(((struct scsi_rw_big *)xs->cmd)->addr);
150 c285a1f8 2020-11-06 stsp -	else if (xs->cmdlen == 16)
151 c285a1f8 2020-11-06 stsp -		blk = _8btol(((struct scsi_rw_16 *)xs->cmd)->addr);
152 c285a1f8 2020-11-06 stsp -	else if (xs->cmdlen == 6)
153 c285a1f8 2020-11-06 stsp -		blk = _3btol(((struct scsi_rw *)xs->cmd)->addr);
154 c285a1f8 2020-11-06 stsp -
155 c285a1f8 2020-11-06 stsp +	blkno = wu->swu_blk_start;
156 c285a1f8 2020-11-06 stsp  	n = xs->datalen >> DEV_BSHIFT;
157 c285a1f8 2020-11-06 stsp  
158 c285a1f8 2020-11-06 stsp  	/*
159 c285a1f8 2020-11-06 stsp  	 * We preallocated enough crypto descs for up to MAXPHYS of I/O.
160 c285a1f8 2020-11-06 stsp -	 * Since there may be less than that we need to tweak the linked list
161 c285a1f8 2020-11-06 stsp +	 * Since there may be less than that we need to tweak the amount
162 c285a1f8 2020-11-06 stsp  	 * of crypto desc structures to be just long enough for our needs.
163 c285a1f8 2020-11-06 stsp  	 */
164 c285a1f8 2020-11-06 stsp -	crd = crwu->cr_descs;
165 c285a1f8 2020-11-06 stsp -	for (i = 0; i < ((MAXPHYS >> DEV_BSHIFT) - n); i++) {
166 c285a1f8 2020-11-06 stsp -		crd = crd->crd_next;
167 c285a1f8 2020-11-06 stsp -		KASSERT(crd);
168 c285a1f8 2020-11-06 stsp -	}
169 c285a1f8 2020-11-06 stsp -	crwu->cr_crp->crp_desc = crd;
170 c285a1f8 2020-11-06 stsp +	KASSERT(crwu->cr_crp->crp_ndescalloc >= n);
171 c285a1f8 2020-11-06 stsp +	crwu->cr_crp->crp_ndesc = n;
172 c285a1f8 2020-11-06 stsp  	flags = (encrypt ? CRD_F_ENCRYPT : 0) |
173 c285a1f8 2020-11-06 stsp  	    CRD_F_IV_PRESENT | CRD_F_IV_EXPLICIT;
174 c285a1f8 2020-11-06 stsp  
175 c285a1f8 2020-11-06 stsp -	/* Select crypto session based on block number */
176 c285a1f8 2020-11-06 stsp -	keyndx = blk >> SR_CRYPTO_KEY_BLKSHIFT;
177 c285a1f8 2020-11-06 stsp -	if (keyndx >= SR_CRYPTO_MAXKEYS)
178 c285a1f8 2020-11-06 stsp -		goto unwind;
179 c285a1f8 2020-11-06 stsp +	/*
180 c285a1f8 2020-11-06 stsp +	 * Select crypto session based on block number.
181 c285a1f8 2020-11-06 stsp +	 *
182 c285a1f8 2020-11-06 stsp +	 * XXX - this does not handle the case where the read/write spans
183 c285a1f8 2020-11-06 stsp +	 * across a different key blocks (e.g. 0.5TB boundary). Currently
184 c285a1f8 2020-11-06 stsp +	 * this is already broken by the use of scr_key[0] below.
185 c285a1f8 2020-11-06 stsp +	 */
186 c285a1f8 2020-11-06 stsp +	keyndx = blkno >> SR_CRYPTO_KEY_BLKSHIFT;
187 c285a1f8 2020-11-06 stsp  	crwu->cr_crp->crp_sid = sd->mds.mdd_crypto.scr_sid[keyndx];
188 c285a1f8 2020-11-06 stsp -	if (crwu->cr_crp->crp_sid == (u_int64_t)-1)
189 c285a1f8 2020-11-06 stsp -		goto unwind;
190 c285a1f8 2020-11-06 stsp  
191 c285a1f8 2020-11-06 stsp +	crwu->cr_crp->crp_opaque = crwu;
192 c285a1f8 2020-11-06 stsp  	crwu->cr_crp->crp_ilen = xs->datalen;
193 c285a1f8 2020-11-06 stsp  	crwu->cr_crp->crp_alloctype = M_DEVBUF;
194 c285a1f8 2020-11-06 stsp +	crwu->cr_crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_NOQUEUE;
195 c285a1f8 2020-11-06 stsp  	crwu->cr_crp->crp_buf = &crwu->cr_uio;
196 c285a1f8 2020-11-06 stsp -	for (i = 0, crd = crwu->cr_crp->crp_desc; crd;
197 c285a1f8 2020-11-06 stsp -	    i++, blk++, crd = crd->crd_next) {
198 c285a1f8 2020-11-06 stsp +	for (i = 0; i < crwu->cr_crp->crp_ndesc; i++, blkno++) {
199 c285a1f8 2020-11-06 stsp +		crd = &crwu->cr_crp->crp_desc[i];
200 c285a1f8 2020-11-06 stsp  		crd->crd_skip = i << DEV_BSHIFT;
201 c285a1f8 2020-11-06 stsp  		crd->crd_len = DEV_BSIZE;
202 c285a1f8 2020-11-06 stsp  		crd->crd_inject = 0;
203 c285a1f8 2020-11-06 stsp  		crd->crd_flags = flags;
204 c285a1f8 2020-11-06 stsp -		crd->crd_alg = CRYPTO_AES_XTS;
205 c285a1f8 2020-11-06 stsp -
206 c285a1f8 2020-11-06 stsp -		switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
207 c285a1f8 2020-11-06 stsp -		case SR_CRYPTOA_AES_XTS_128:
208 c285a1f8 2020-11-06 stsp -			crd->crd_klen = 256;
209 c285a1f8 2020-11-06 stsp -			break;
210 c285a1f8 2020-11-06 stsp -		case SR_CRYPTOA_AES_XTS_256:
211 c285a1f8 2020-11-06 stsp -			crd->crd_klen = 512;
212 c285a1f8 2020-11-06 stsp -			break;
213 c285a1f8 2020-11-06 stsp -		default:
214 c285a1f8 2020-11-06 stsp -			goto unwind;
215 c285a1f8 2020-11-06 stsp -		}
216 c285a1f8 2020-11-06 stsp +		crd->crd_alg = sd->mds.mdd_crypto.scr_alg;
217 c285a1f8 2020-11-06 stsp +		crd->crd_klen = sd->mds.mdd_crypto.scr_klen;
218 c285a1f8 2020-11-06 stsp  		crd->crd_key = sd->mds.mdd_crypto.scr_key[0];
219 c285a1f8 2020-11-06 stsp -		bcopy(&blk, crd->crd_iv, sizeof(blk));
220 c285a1f8 2020-11-06 stsp +		memcpy(crd->crd_iv, &blkno, sizeof(blkno));
221 c285a1f8 2020-11-06 stsp  	}
222 c285a1f8 2020-11-06 stsp -	crwu->cr_wu = wu;
223 c285a1f8 2020-11-06 stsp -	crwu->cr_crp->crp_opaque = crwu;
224 c285a1f8 2020-11-06 stsp  
225 c285a1f8 2020-11-06 stsp  	return (crwu);
226 c285a1f8 2020-11-06 stsp -
227 c285a1f8 2020-11-06 stsp -unwind:
228 c285a1f8 2020-11-06 stsp -	/* steal the descriptors back from the cryptop */
229 c285a1f8 2020-11-06 stsp -	crwu->cr_crp->crp_desc = NULL;
230 c285a1f8 2020-11-06 stsp -
231 c285a1f8 2020-11-06 stsp -	return (NULL);
232 405e89a6 2022-08-02 op -}
233 405e89a6 2022-08-02 op -
234 c285a1f8 2020-11-06 stsp -void
235 c285a1f8 2020-11-06 stsp -sr_crypto_wu_put(struct sr_crypto_wu *crwu)
236 c285a1f8 2020-11-06 stsp -{
237 c285a1f8 2020-11-06 stsp -	struct cryptop		*crp = crwu->cr_crp;
238 c285a1f8 2020-11-06 stsp -	struct sr_workunit	*wu = crwu->cr_wu;
239 c285a1f8 2020-11-06 stsp -	struct sr_discipline	*sd = wu->swu_dis;
240 c285a1f8 2020-11-06 stsp -
241 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_wu_put crwu: %p\n",
242 c285a1f8 2020-11-06 stsp -	    DEVNAME(wu->swu_dis->sd_sc), crwu);
243 c285a1f8 2020-11-06 stsp -
244 c285a1f8 2020-11-06 stsp -	/* steal the descriptors back from the cryptop */
245 c285a1f8 2020-11-06 stsp -	crp->crp_desc = NULL;
246 c285a1f8 2020-11-06 stsp -
247 c285a1f8 2020-11-06 stsp -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
248 c285a1f8 2020-11-06 stsp -	TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
249 c285a1f8 2020-11-06 stsp -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
250 405e89a6 2022-08-02 op  }
251 405e89a6 2022-08-02 op  
252 c285a1f8 2020-11-06 stsp  int
253 c285a1f8 2020-11-06 stsp @@ -386,9 +339,8 @@
254 c285a1f8 2020-11-06 stsp  		if (sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint) <
255 c285a1f8 2020-11-06 stsp  		    kdfinfo->genkdf.len)
256 c285a1f8 2020-11-06 stsp  			goto out;
257 c285a1f8 2020-11-06 stsp -		bcopy(&kdfinfo->genkdf,
258 c285a1f8 2020-11-06 stsp -		    sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
259 c285a1f8 2020-11-06 stsp -		    kdfinfo->genkdf.len);
260 c285a1f8 2020-11-06 stsp +		memcpy(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
261 c285a1f8 2020-11-06 stsp +		    &kdfinfo->genkdf, kdfinfo->genkdf.len);
262 c285a1f8 2020-11-06 stsp  	}
263 c285a1f8 2020-11-06 stsp  
264 c285a1f8 2020-11-06 stsp  	/* copy mask key to run-time meta data */
265 c285a1f8 2020-11-06 stsp @@ -396,7 +348,7 @@
266 c285a1f8 2020-11-06 stsp  		if (sizeof(sd->mds.mdd_crypto.scr_maskkey) <
267 c285a1f8 2020-11-06 stsp  		    sizeof(kdfinfo->maskkey))
268 c285a1f8 2020-11-06 stsp  			goto out;
269 c285a1f8 2020-11-06 stsp -		bcopy(&kdfinfo->maskkey, sd->mds.mdd_crypto.scr_maskkey,
270 c285a1f8 2020-11-06 stsp +		memcpy(sd->mds.mdd_crypto.scr_maskkey, &kdfinfo->maskkey,
271 c285a1f8 2020-11-06 stsp  		    sizeof(kdfinfo->maskkey));
272 c285a1f8 2020-11-06 stsp  	}
273 c285a1f8 2020-11-06 stsp  
274 c285a1f8 2020-11-06 stsp @@ -404,7 +356,7 @@
275 c285a1f8 2020-11-06 stsp  	rv = 0;
276 c285a1f8 2020-11-06 stsp  out:
277 c285a1f8 2020-11-06 stsp  	explicit_bzero(kdfinfo, bc->bc_opaque_size);
278 c285a1f8 2020-11-06 stsp -	free(kdfinfo, M_DEVBUF);
279 c285a1f8 2020-11-06 stsp +	free(kdfinfo, M_DEVBUF, bc->bc_opaque_size);
280 c285a1f8 2020-11-06 stsp  
281 c285a1f8 2020-11-06 stsp  	return (rv);
282 c285a1f8 2020-11-06 stsp  }
283 c285a1f8 2020-11-06 stsp @@ -424,7 +376,7 @@
284 c285a1f8 2020-11-06 stsp  		rv = 0;
285 c285a1f8 2020-11-06 stsp  		break;
286 c285a1f8 2020-11-06 stsp  	default:
287 c285a1f8 2020-11-06 stsp -		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n",
288 c285a1f8 2020-11-06 stsp +		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %d\n",
289 c285a1f8 2020-11-06 stsp  		    "softraid", alg);
290 c285a1f8 2020-11-06 stsp  		rv = -1;
291 c285a1f8 2020-11-06 stsp  		goto out;
292 c285a1f8 2020-11-06 stsp @@ -450,7 +402,7 @@
293 c285a1f8 2020-11-06 stsp  		rv = 0;
294 c285a1f8 2020-11-06 stsp  		break;
295 c285a1f8 2020-11-06 stsp  	default:
296 c285a1f8 2020-11-06 stsp -		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %u\n",
297 c285a1f8 2020-11-06 stsp +		DNPRINTF(SR_D_DIS, "%s: unsupported encryption algorithm %d\n",
298 c285a1f8 2020-11-06 stsp  		    "softraid", alg);
299 c285a1f8 2020-11-06 stsp  		rv = -1;
300 c285a1f8 2020-11-06 stsp  		goto out;
301 405e89a6 2022-08-02 op @@ -615,6 +567,17 @@
302 405e89a6 2022-08-02 op  		sr_error(sd->sd_sc, "incorrect key or passphrase");
303 405e89a6 2022-08-02 op  		rv = EPERM;
304 c285a1f8 2020-11-06 stsp  		goto out;
305 405e89a6 2022-08-02 op +	}
306 405e89a6 2022-08-02 op +
307 c285a1f8 2020-11-06 stsp +	/* Copy new KDF hint to metadata, if supplied. */
308 c285a1f8 2020-11-06 stsp +	if (kdfinfo2->flags & SR_CRYPTOKDF_HINT) {
309 c285a1f8 2020-11-06 stsp +		if (kdfinfo2->genkdf.len >
310 c285a1f8 2020-11-06 stsp +		    sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint))
311 c285a1f8 2020-11-06 stsp +			goto out;
312 c285a1f8 2020-11-06 stsp +		explicit_bzero(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
313 c285a1f8 2020-11-06 stsp +		    sizeof(sd->mds.mdd_crypto.scr_meta->scm_kdfhint));
314 c285a1f8 2020-11-06 stsp +		memcpy(sd->mds.mdd_crypto.scr_meta->scm_kdfhint,
315 c285a1f8 2020-11-06 stsp +		    &kdfinfo2->genkdf, kdfinfo2->genkdf.len);
316 405e89a6 2022-08-02 op  	}
317 405e89a6 2022-08-02 op  
318 c285a1f8 2020-11-06 stsp  	/* Mask the disk keys. */
319 c285a1f8 2020-11-06 stsp @@ -630,7 +593,7 @@
320 c285a1f8 2020-11-06 stsp  	    sizeof(sd->mds.mdd_crypto.scr_key), check_digest);
321 c285a1f8 2020-11-06 stsp  
322 c285a1f8 2020-11-06 stsp  	/* Copy new encrypted key and HMAC to metadata. */
323 c285a1f8 2020-11-06 stsp -	bcopy(check_digest, sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac,
324 c285a1f8 2020-11-06 stsp +	memcpy(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac, check_digest,
325 c285a1f8 2020-11-06 stsp  	    sizeof(sd->mds.mdd_crypto.scr_meta->chk_hmac_sha1.sch_mac));
326 c285a1f8 2020-11-06 stsp  
327 c285a1f8 2020-11-06 stsp  	rv = 0; /* Success */
328 c285a1f8 2020-11-06 stsp @@ -638,7 +601,7 @@
329 c285a1f8 2020-11-06 stsp  out:
330 c285a1f8 2020-11-06 stsp  	if (p) {
331 c285a1f8 2020-11-06 stsp  		explicit_bzero(p, ksz);
332 c285a1f8 2020-11-06 stsp -		free(p, M_DEVBUF);
333 c285a1f8 2020-11-06 stsp +		free(p, M_DEVBUF, ksz);
334 c285a1f8 2020-11-06 stsp  	}
335 c285a1f8 2020-11-06 stsp  
336 c285a1f8 2020-11-06 stsp  	explicit_bzero(check_digest, sizeof(check_digest));
337 c285a1f8 2020-11-06 stsp @@ -686,7 +649,7 @@
338 c285a1f8 2020-11-06 stsp  		DNPRINTF(SR_D_META,"%s: sr_crypto_create_key_disk cannot "
339 c285a1f8 2020-11-06 stsp  		    "open %s\n", DEVNAME(sc), devname);
340 c285a1f8 2020-11-06 stsp  		vput(vn);
341 c285a1f8 2020-11-06 stsp -		goto fail;
342 c285a1f8 2020-11-06 stsp +		goto done;
343 c285a1f8 2020-11-06 stsp  	}
344 c285a1f8 2020-11-06 stsp  	open = 1; /* close dev on error */
345 c285a1f8 2020-11-06 stsp  
346 c285a1f8 2020-11-06 stsp @@ -696,19 +659,12 @@
347 c285a1f8 2020-11-06 stsp  	    FREAD, NOCRED, curproc)) {
348 c285a1f8 2020-11-06 stsp  		DNPRINTF(SR_D_META, "%s: sr_crypto_create_key_disk ioctl "
349 c285a1f8 2020-11-06 stsp  		    "failed\n", DEVNAME(sc));
350 c285a1f8 2020-11-06 stsp -		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
351 c285a1f8 2020-11-06 stsp -		vput(vn);
352 c285a1f8 2020-11-06 stsp -		goto fail;
353 c285a1f8 2020-11-06 stsp +		goto done;
354 c285a1f8 2020-11-06 stsp  	}
355 c285a1f8 2020-11-06 stsp -	if (label.d_secsize != DEV_BSIZE) {
356 c285a1f8 2020-11-06 stsp -		sr_error(sc, "%s has unsupported sector size (%d)",
357 c285a1f8 2020-11-06 stsp -		    devname, label.d_secsize);
358 c285a1f8 2020-11-06 stsp -		goto fail;
359 c285a1f8 2020-11-06 stsp -	}
360 c285a1f8 2020-11-06 stsp  	if (label.d_partitions[part].p_fstype != FS_RAID) {
361 c285a1f8 2020-11-06 stsp -		sr_error(sc, "%s partition not of type RAID (%d)\n",
362 c285a1f8 2020-11-06 stsp +		sr_error(sc, "%s partition not of type RAID (%d)",
363 c285a1f8 2020-11-06 stsp  		    devname, label.d_partitions[part].p_fstype);
364 c285a1f8 2020-11-06 stsp -		goto fail;
365 c285a1f8 2020-11-06 stsp +		goto done;
366 c285a1f8 2020-11-06 stsp  	}
367 c285a1f8 2020-11-06 stsp  
368 c285a1f8 2020-11-06 stsp  	/*
369 c285a1f8 2020-11-06 stsp @@ -728,7 +684,7 @@
370 c285a1f8 2020-11-06 stsp  	km->scmi.scm_size = 0;
371 c285a1f8 2020-11-06 stsp  	km->scmi.scm_coerced_size = 0;
372 c285a1f8 2020-11-06 stsp  	strlcpy(km->scmi.scm_devname, devname, sizeof(km->scmi.scm_devname));
373 c285a1f8 2020-11-06 stsp -	bcopy(&sd->sd_meta->ssdi.ssd_uuid, &km->scmi.scm_uuid,
374 c285a1f8 2020-11-06 stsp +	memcpy(&km->scmi.scm_uuid, &sd->sd_meta->ssdi.ssd_uuid,
375 c285a1f8 2020-11-06 stsp  	    sizeof(struct sr_uuid));
376 c285a1f8 2020-11-06 stsp  
377 c285a1f8 2020-11-06 stsp  	sr_checksum(sc, km, &km->scm_checksum,
378 c285a1f8 2020-11-06 stsp @@ -745,7 +701,7 @@
379 c285a1f8 2020-11-06 stsp  	sm->ssdi.ssd_version = SR_META_VERSION;
380 c285a1f8 2020-11-06 stsp  	sm->ssd_ondisk = 0;
381 c285a1f8 2020-11-06 stsp  	sm->ssdi.ssd_vol_flags = 0;
382 c285a1f8 2020-11-06 stsp -	bcopy(&sd->sd_meta->ssdi.ssd_uuid, &sm->ssdi.ssd_uuid,
383 c285a1f8 2020-11-06 stsp +	memcpy(&sm->ssdi.ssd_uuid, &sd->sd_meta->ssdi.ssd_uuid,
384 c285a1f8 2020-11-06 stsp  	    sizeof(struct sr_uuid));
385 c285a1f8 2020-11-06 stsp  	sm->ssdi.ssd_chunk_no = 1;
386 c285a1f8 2020-11-06 stsp  	sm->ssdi.ssd_volid = SR_KEYDISK_VOLID;
387 c285a1f8 2020-11-06 stsp @@ -785,7 +741,7 @@
388 c285a1f8 2020-11-06 stsp  	omi->omi_som->som_type = SR_OPT_KEYDISK;
389 c285a1f8 2020-11-06 stsp  	omi->omi_som->som_length = sizeof(struct sr_meta_keydisk);
390 c285a1f8 2020-11-06 stsp  	skm = (struct sr_meta_keydisk *)omi->omi_som;
391 c285a1f8 2020-11-06 stsp -	bcopy(sd->mds.mdd_crypto.scr_maskkey, &skm->skm_maskkey,
392 c285a1f8 2020-11-06 stsp +	memcpy(&skm->skm_maskkey, sd->mds.mdd_crypto.scr_maskkey,
393 c285a1f8 2020-11-06 stsp  	    sizeof(skm->skm_maskkey));
394 c285a1f8 2020-11-06 stsp  	SLIST_INSERT_HEAD(&fakesd->sd_meta_opt, omi, omi_link);
395 c285a1f8 2020-11-06 stsp  	fakesd->sd_meta->ssdi.ssd_opt_no++;
396 c285a1f8 2020-11-06 stsp @@ -799,19 +755,16 @@
397 c285a1f8 2020-11-06 stsp  	goto done;
398 c285a1f8 2020-11-06 stsp  
399 c285a1f8 2020-11-06 stsp  fail:
400 c285a1f8 2020-11-06 stsp -	if (key_disk)
401 c285a1f8 2020-11-06 stsp -		free(key_disk, M_DEVBUF);
402 c285a1f8 2020-11-06 stsp +	free(key_disk, M_DEVBUF, sizeof(struct sr_chunk));
403 c285a1f8 2020-11-06 stsp  	key_disk = NULL;
404 c285a1f8 2020-11-06 stsp  
405 c285a1f8 2020-11-06 stsp  done:
406 c285a1f8 2020-11-06 stsp -	if (omi)
407 c285a1f8 2020-11-06 stsp -		free(omi, M_DEVBUF);
408 c285a1f8 2020-11-06 stsp +	free(omi, M_DEVBUF, sizeof(struct sr_meta_opt_item));
409 c285a1f8 2020-11-06 stsp  	if (fakesd && fakesd->sd_vol.sv_chunks)
410 c285a1f8 2020-11-06 stsp -		free(fakesd->sd_vol.sv_chunks, M_DEVBUF);
411 c285a1f8 2020-11-06 stsp -	if (fakesd)
412 c285a1f8 2020-11-06 stsp -		free(fakesd, M_DEVBUF);
413 c285a1f8 2020-11-06 stsp -	if (sm)
414 c285a1f8 2020-11-06 stsp -		free(sm, M_DEVBUF);
415 c285a1f8 2020-11-06 stsp +		free(fakesd->sd_vol.sv_chunks, M_DEVBUF,
416 c285a1f8 2020-11-06 stsp +		    sizeof(struct sr_chunk *));
417 c285a1f8 2020-11-06 stsp +	free(fakesd, M_DEVBUF, sizeof(struct sr_discipline));
418 c285a1f8 2020-11-06 stsp +	free(sm, M_DEVBUF, sizeof(struct sr_metadata));
419 c285a1f8 2020-11-06 stsp  	if (open) {
420 c285a1f8 2020-11-06 stsp  		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
421 c285a1f8 2020-11-06 stsp  		vput(vn);
422 c285a1f8 2020-11-06 stsp @@ -855,7 +808,7 @@
423 c285a1f8 2020-11-06 stsp  		sr_error(sc, "cannot open key disk %s", devname);
424 c285a1f8 2020-11-06 stsp  		goto done;
425 c285a1f8 2020-11-06 stsp  	}
426 c285a1f8 2020-11-06 stsp -	if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) {
427 c285a1f8 2020-11-06 stsp +	if (VOP_OPEN(vn, FREAD, NOCRED, curproc)) {
428 c285a1f8 2020-11-06 stsp  		DNPRINTF(SR_D_META,"%s: sr_crypto_read_key_disk cannot "
429 c285a1f8 2020-11-06 stsp  		    "open %s\n", DEVNAME(sc), devname);
430 c285a1f8 2020-11-06 stsp  		vput(vn);
431 c285a1f8 2020-11-06 stsp @@ -869,17 +822,10 @@
432 c285a1f8 2020-11-06 stsp  	    NOCRED, curproc)) {
433 c285a1f8 2020-11-06 stsp  		DNPRINTF(SR_D_META, "%s: sr_crypto_read_key_disk ioctl "
434 c285a1f8 2020-11-06 stsp  		    "failed\n", DEVNAME(sc));
435 c285a1f8 2020-11-06 stsp -		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
436 c285a1f8 2020-11-06 stsp -		vput(vn);
437 c285a1f8 2020-11-06 stsp  		goto done;
438 c285a1f8 2020-11-06 stsp  	}
439 c285a1f8 2020-11-06 stsp -	if (label.d_secsize != DEV_BSIZE) {
440 c285a1f8 2020-11-06 stsp -		sr_error(sc, "%s has unsupported sector size (%d)",
441 c285a1f8 2020-11-06 stsp -		    devname, label.d_secsize);
442 c285a1f8 2020-11-06 stsp -		goto done;
443 c285a1f8 2020-11-06 stsp -	}
444 c285a1f8 2020-11-06 stsp  	if (label.d_partitions[part].p_fstype != FS_RAID) {
445 c285a1f8 2020-11-06 stsp -		sr_error(sc, "%s partition not of type RAID (%d)\n",
446 c285a1f8 2020-11-06 stsp +		sr_error(sc, "%s partition not of type RAID (%d)",
447 c285a1f8 2020-11-06 stsp  		    devname, label.d_partitions[part].p_fstype);
448 c285a1f8 2020-11-06 stsp  		goto done;
449 c285a1f8 2020-11-06 stsp  	}
450 c285a1f8 2020-11-06 stsp @@ -887,7 +833,7 @@
451 c285a1f8 2020-11-06 stsp  	/*
452 c285a1f8 2020-11-06 stsp  	 * Read and validate key disk metadata.
453 c285a1f8 2020-11-06 stsp  	 */
454 c285a1f8 2020-11-06 stsp -	sm = malloc(SR_META_SIZE * 512, M_DEVBUF, M_WAITOK | M_ZERO);
455 c285a1f8 2020-11-06 stsp +	sm = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_WAITOK | M_ZERO);
456 c285a1f8 2020-11-06 stsp  	if (sr_meta_native_read(sd, dev, sm, NULL)) {
457 c285a1f8 2020-11-06 stsp  		sr_error(sc, "native bootprobe could not read native metadata");
458 c285a1f8 2020-11-06 stsp  		goto done;
459 c285a1f8 2020-11-06 stsp @@ -911,7 +857,7 @@
460 c285a1f8 2020-11-06 stsp  	key_disk->src_vn = vn;
461 c285a1f8 2020-11-06 stsp  	key_disk->src_size = 0;
462 c285a1f8 2020-11-06 stsp  
463 c285a1f8 2020-11-06 stsp -	bcopy((struct sr_meta_chunk *)(sm + 1), &key_disk->src_meta,
464 c285a1f8 2020-11-06 stsp +	memcpy(&key_disk->src_meta, (struct sr_meta_chunk *)(sm + 1),
465 c285a1f8 2020-11-06 stsp  	    sizeof(key_disk->src_meta));
466 c285a1f8 2020-11-06 stsp  
467 c285a1f8 2020-11-06 stsp  	/* Read mask key from optional metadata. */
468 c285a1f8 2020-11-06 stsp @@ -920,13 +866,12 @@
469 c285a1f8 2020-11-06 stsp  		omh = omi->omi_som;
470 c285a1f8 2020-11-06 stsp  		if (omh->som_type == SR_OPT_KEYDISK) {
471 c285a1f8 2020-11-06 stsp  			skm = (struct sr_meta_keydisk *)omh;
472 c285a1f8 2020-11-06 stsp -			bcopy(&skm->skm_maskkey,
473 c285a1f8 2020-11-06 stsp -			    sd->mds.mdd_crypto.scr_maskkey,
474 c285a1f8 2020-11-06 stsp +			memcpy(sd->mds.mdd_crypto.scr_maskkey, &skm->skm_maskkey,
475 c285a1f8 2020-11-06 stsp  			    sizeof(sd->mds.mdd_crypto.scr_maskkey));
476 c285a1f8 2020-11-06 stsp  		} else if (omh->som_type == SR_OPT_CRYPTO) {
477 c285a1f8 2020-11-06 stsp  			/* Original keydisk format with key in crypto area. */
478 c285a1f8 2020-11-06 stsp -			bcopy(omh + sizeof(struct sr_meta_opt_hdr),
479 c285a1f8 2020-11-06 stsp -			    sd->mds.mdd_crypto.scr_maskkey,
480 c285a1f8 2020-11-06 stsp +			memcpy(sd->mds.mdd_crypto.scr_maskkey,
481 c285a1f8 2020-11-06 stsp +			    omh + sizeof(struct sr_meta_opt_hdr),
482 c285a1f8 2020-11-06 stsp  			    sizeof(sd->mds.mdd_crypto.scr_maskkey));
483 c285a1f8 2020-11-06 stsp  		}
484 c285a1f8 2020-11-06 stsp  	}
485 c285a1f8 2020-11-06 stsp @@ -934,15 +879,13 @@
486 c285a1f8 2020-11-06 stsp  	open = 0;
487 c285a1f8 2020-11-06 stsp  
488 c285a1f8 2020-11-06 stsp  done:
489 c285a1f8 2020-11-06 stsp -	for (omi = SLIST_FIRST(&som); omi != SLIST_END(&som); omi = omi_next) {
490 c285a1f8 2020-11-06 stsp +	for (omi = SLIST_FIRST(&som); omi != NULL; omi = omi_next) {
491 c285a1f8 2020-11-06 stsp  		omi_next = SLIST_NEXT(omi, omi_link);
492 c285a1f8 2020-11-06 stsp -		if (omi->omi_som)
493 c285a1f8 2020-11-06 stsp -			free(omi->omi_som, M_DEVBUF);
494 c285a1f8 2020-11-06 stsp -		free(omi, M_DEVBUF);
495 c285a1f8 2020-11-06 stsp +		free(omi->omi_som, M_DEVBUF, 0);
496 c285a1f8 2020-11-06 stsp +		free(omi, M_DEVBUF, sizeof(struct sr_meta_opt_item));
497 c285a1f8 2020-11-06 stsp  	}
498 c285a1f8 2020-11-06 stsp  
499 c285a1f8 2020-11-06 stsp -	if (sm)
500 c285a1f8 2020-11-06 stsp -		free(sm, M_DEVBUF);
501 c285a1f8 2020-11-06 stsp +	free(sm, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
502 c285a1f8 2020-11-06 stsp  
503 c285a1f8 2020-11-06 stsp  	if (vn && open) {
504 c285a1f8 2020-11-06 stsp  		VOP_CLOSE(vn, FREAD, NOCRED, curproc);
505 405e89a6 2022-08-02 op @@ -950,18 +893,45 @@
506 405e89a6 2022-08-02 op  	}
507 c285a1f8 2020-11-06 stsp  
508 405e89a6 2022-08-02 op  	return key_disk;
509 405e89a6 2022-08-02 op +}
510 405e89a6 2022-08-02 op +
511 c285a1f8 2020-11-06 stsp +static void
512 c285a1f8 2020-11-06 stsp +sr_crypto_free_sessions(struct sr_discipline *sd)
513 c285a1f8 2020-11-06 stsp +{
514 c285a1f8 2020-11-06 stsp +	u_int			i;
515 c285a1f8 2020-11-06 stsp +
516 c285a1f8 2020-11-06 stsp +	for (i = 0; i < SR_CRYPTO_MAXKEYS; i++) {
517 c285a1f8 2020-11-06 stsp +		if (sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1) {
518 c285a1f8 2020-11-06 stsp +			crypto_freesession(sd->mds.mdd_crypto.scr_sid[i]);
519 c285a1f8 2020-11-06 stsp +			sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
520 c285a1f8 2020-11-06 stsp +		}
521 c285a1f8 2020-11-06 stsp +	}
522 405e89a6 2022-08-02 op  }
523 405e89a6 2022-08-02 op  
524 c285a1f8 2020-11-06 stsp  int
525 c285a1f8 2020-11-06 stsp  sr_crypto_alloc_resources(struct sr_discipline *sd)
526 c285a1f8 2020-11-06 stsp  {
527 c285a1f8 2020-11-06 stsp -	struct cryptoini	cri;
528 c285a1f8 2020-11-06 stsp +	struct sr_workunit	*wu;
529 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu;
530 c285a1f8 2020-11-06 stsp +	struct cryptoini	cri;
531 c285a1f8 2020-11-06 stsp  	u_int			num_keys, i;
532 c285a1f8 2020-11-06 stsp  
533 c285a1f8 2020-11-06 stsp  	DNPRINTF(SR_D_DIS, "%s: sr_crypto_alloc_resources\n",
534 c285a1f8 2020-11-06 stsp  	    DEVNAME(sd->sd_sc));
535 c285a1f8 2020-11-06 stsp  
536 c285a1f8 2020-11-06 stsp +	sd->mds.mdd_crypto.scr_alg = CRYPTO_AES_XTS;
537 c285a1f8 2020-11-06 stsp +	switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
538 c285a1f8 2020-11-06 stsp +	case SR_CRYPTOA_AES_XTS_128:
539 c285a1f8 2020-11-06 stsp +		sd->mds.mdd_crypto.scr_klen = 256;
540 c285a1f8 2020-11-06 stsp +		break;
541 c285a1f8 2020-11-06 stsp +	case SR_CRYPTOA_AES_XTS_256:
542 c285a1f8 2020-11-06 stsp +		sd->mds.mdd_crypto.scr_klen = 512;
543 c285a1f8 2020-11-06 stsp +		break;
544 c285a1f8 2020-11-06 stsp +	default:
545 c285a1f8 2020-11-06 stsp +		sr_error(sd->sd_sc, "unknown crypto algorithm");
546 c285a1f8 2020-11-06 stsp +		return (EINVAL);
547 c285a1f8 2020-11-06 stsp +	}
548 c285a1f8 2020-11-06 stsp +
549 c285a1f8 2020-11-06 stsp  	for (i = 0; i < SR_CRYPTO_MAXKEYS; i++)
550 c285a1f8 2020-11-06 stsp  		sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
551 c285a1f8 2020-11-06 stsp  
552 c285a1f8 2020-11-06 stsp @@ -979,61 +949,34 @@
553 c285a1f8 2020-11-06 stsp  	}
554 c285a1f8 2020-11-06 stsp  
555 c285a1f8 2020-11-06 stsp  	/*
556 c285a1f8 2020-11-06 stsp -	 * For each wu allocate the uio, iovec and crypto structures.
557 c285a1f8 2020-11-06 stsp -	 * these have to be allocated now because during runtime we can't
558 c285a1f8 2020-11-06 stsp -	 * fail an allocation without failing the io (which can cause real
559 c285a1f8 2020-11-06 stsp +	 * For each work unit allocate the uio, iovec and crypto structures.
560 c285a1f8 2020-11-06 stsp +	 * These have to be allocated now because during runtime we cannot
561 c285a1f8 2020-11-06 stsp +	 * fail an allocation without failing the I/O (which can cause real
562 c285a1f8 2020-11-06 stsp  	 * problems).
563 c285a1f8 2020-11-06 stsp  	 */
564 c285a1f8 2020-11-06 stsp -	mtx_init(&sd->mds.mdd_crypto.scr_mutex, IPL_BIO);
565 c285a1f8 2020-11-06 stsp -	TAILQ_INIT(&sd->mds.mdd_crypto.scr_wus);
566 c285a1f8 2020-11-06 stsp -	for (i = 0; i < sd->sd_max_wu; i++) {
567 c285a1f8 2020-11-06 stsp -		crwu = malloc(sizeof(*crwu), M_DEVBUF,
568 c285a1f8 2020-11-06 stsp -		    M_WAITOK | M_ZERO | M_CANFAIL);
569 c285a1f8 2020-11-06 stsp -		if (crwu == NULL)
570 c285a1f8 2020-11-06 stsp -		    return (ENOMEM);
571 c285a1f8 2020-11-06 stsp -		/* put it on the list now so if we fail it'll be freed */
572 c285a1f8 2020-11-06 stsp -		mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
573 c285a1f8 2020-11-06 stsp -		TAILQ_INSERT_TAIL(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
574 c285a1f8 2020-11-06 stsp -		mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
575 c285a1f8 2020-11-06 stsp -
576 c285a1f8 2020-11-06 stsp +	TAILQ_FOREACH(wu, &sd->sd_wu, swu_next) {
577 c285a1f8 2020-11-06 stsp +		crwu = (struct sr_crypto_wu *)wu;
578 c285a1f8 2020-11-06 stsp  		crwu->cr_uio.uio_iov = &crwu->cr_iov;
579 c285a1f8 2020-11-06 stsp  		crwu->cr_dmabuf = dma_alloc(MAXPHYS, PR_WAITOK);
580 c285a1f8 2020-11-06 stsp  		crwu->cr_crp = crypto_getreq(MAXPHYS >> DEV_BSHIFT);
581 c285a1f8 2020-11-06 stsp  		if (crwu->cr_crp == NULL)
582 c285a1f8 2020-11-06 stsp  			return (ENOMEM);
583 c285a1f8 2020-11-06 stsp -		/* steal the list of cryptodescs */
584 c285a1f8 2020-11-06 stsp -		crwu->cr_descs = crwu->cr_crp->crp_desc;
585 c285a1f8 2020-11-06 stsp -		crwu->cr_crp->crp_desc = NULL;
586 c285a1f8 2020-11-06 stsp  	}
587 c285a1f8 2020-11-06 stsp  
588 c285a1f8 2020-11-06 stsp -	bzero(&cri, sizeof(cri));
589 c285a1f8 2020-11-06 stsp -	cri.cri_alg = CRYPTO_AES_XTS;
590 c285a1f8 2020-11-06 stsp -	switch (sd->mds.mdd_crypto.scr_meta->scm_alg) {
591 c285a1f8 2020-11-06 stsp -	case SR_CRYPTOA_AES_XTS_128:
592 c285a1f8 2020-11-06 stsp -		cri.cri_klen = 256;
593 c285a1f8 2020-11-06 stsp -		break;
594 c285a1f8 2020-11-06 stsp -	case SR_CRYPTOA_AES_XTS_256:
595 c285a1f8 2020-11-06 stsp -		cri.cri_klen = 512;
596 c285a1f8 2020-11-06 stsp -		break;
597 c285a1f8 2020-11-06 stsp -	default:
598 c285a1f8 2020-11-06 stsp -		return (EINVAL);
599 c285a1f8 2020-11-06 stsp -	}
600 c285a1f8 2020-11-06 stsp +	memset(&cri, 0, sizeof(cri));
601 c285a1f8 2020-11-06 stsp +	cri.cri_alg = sd->mds.mdd_crypto.scr_alg;
602 c285a1f8 2020-11-06 stsp +	cri.cri_klen = sd->mds.mdd_crypto.scr_klen;
603 c285a1f8 2020-11-06 stsp  
604 c285a1f8 2020-11-06 stsp -	/* Allocate a session for every 2^SR_CRYPTO_KEY_BLKSHIFT blocks */
605 c285a1f8 2020-11-06 stsp -	num_keys = sd->sd_meta->ssdi.ssd_size >> SR_CRYPTO_KEY_BLKSHIFT;
606 c285a1f8 2020-11-06 stsp -	if (num_keys >= SR_CRYPTO_MAXKEYS)
607 c285a1f8 2020-11-06 stsp +	/* Allocate a session for every 2^SR_CRYPTO_KEY_BLKSHIFT blocks. */
608 c285a1f8 2020-11-06 stsp +	num_keys = ((sd->sd_meta->ssdi.ssd_size - 1) >>
609 c285a1f8 2020-11-06 stsp +	    SR_CRYPTO_KEY_BLKSHIFT) + 1;
610 c285a1f8 2020-11-06 stsp +	if (num_keys > SR_CRYPTO_MAXKEYS)
611 c285a1f8 2020-11-06 stsp  		return (EFBIG);
612 c285a1f8 2020-11-06 stsp -	for (i = 0; i <= num_keys; i++) {
613 c285a1f8 2020-11-06 stsp +	for (i = 0; i < num_keys; i++) {
614 c285a1f8 2020-11-06 stsp  		cri.cri_key = sd->mds.mdd_crypto.scr_key[i];
615 c285a1f8 2020-11-06 stsp  		if (crypto_newsession(&sd->mds.mdd_crypto.scr_sid[i],
616 c285a1f8 2020-11-06 stsp  		    &cri, 0) != 0) {
617 c285a1f8 2020-11-06 stsp -			for (i = 0;
618 c285a1f8 2020-11-06 stsp -			     sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1;
619 c285a1f8 2020-11-06 stsp -			     i++) {
620 c285a1f8 2020-11-06 stsp -				crypto_freesession(
621 c285a1f8 2020-11-06 stsp -				    sd->mds.mdd_crypto.scr_sid[i]);
622 c285a1f8 2020-11-06 stsp -				sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
623 c285a1f8 2020-11-06 stsp -			}
624 c285a1f8 2020-11-06 stsp +			sr_crypto_free_sessions(sd);
625 c285a1f8 2020-11-06 stsp  			return (EINVAL);
626 c285a1f8 2020-11-06 stsp  		}
627 c285a1f8 2020-11-06 stsp  	}
628 c285a1f8 2020-11-06 stsp @@ -1046,39 +989,30 @@
629 c285a1f8 2020-11-06 stsp  void
630 c285a1f8 2020-11-06 stsp  sr_crypto_free_resources(struct sr_discipline *sd)
631 c285a1f8 2020-11-06 stsp  {
632 c285a1f8 2020-11-06 stsp +	struct sr_workunit	*wu;
633 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu;
634 c285a1f8 2020-11-06 stsp -	u_int			i;
635 c285a1f8 2020-11-06 stsp  
636 c285a1f8 2020-11-06 stsp  	DNPRINTF(SR_D_DIS, "%s: sr_crypto_free_resources\n",
637 c285a1f8 2020-11-06 stsp  	    DEVNAME(sd->sd_sc));
638 c285a1f8 2020-11-06 stsp  
639 c285a1f8 2020-11-06 stsp  	if (sd->mds.mdd_crypto.key_disk != NULL) {
640 c285a1f8 2020-11-06 stsp -		explicit_bzero(sd->mds.mdd_crypto.key_disk, sizeof
641 c285a1f8 2020-11-06 stsp -		    sd->mds.mdd_crypto.key_disk);
642 c285a1f8 2020-11-06 stsp -		free(sd->mds.mdd_crypto.key_disk, M_DEVBUF);
643 c285a1f8 2020-11-06 stsp +		explicit_bzero(sd->mds.mdd_crypto.key_disk,
644 c285a1f8 2020-11-06 stsp +		    sizeof(*sd->mds.mdd_crypto.key_disk));
645 c285a1f8 2020-11-06 stsp +		free(sd->mds.mdd_crypto.key_disk, M_DEVBUF,
646 c285a1f8 2020-11-06 stsp +		    sizeof(*sd->mds.mdd_crypto.key_disk));
647 c285a1f8 2020-11-06 stsp  	}
648 c285a1f8 2020-11-06 stsp  
649 c285a1f8 2020-11-06 stsp  	sr_hotplug_unregister(sd, sr_crypto_hotplug);
650 c285a1f8 2020-11-06 stsp  
651 c285a1f8 2020-11-06 stsp -	for (i = 0; sd->mds.mdd_crypto.scr_sid[i] != (u_int64_t)-1; i++) {
652 c285a1f8 2020-11-06 stsp -		crypto_freesession(sd->mds.mdd_crypto.scr_sid[i]);
653 c285a1f8 2020-11-06 stsp -		sd->mds.mdd_crypto.scr_sid[i] = (u_int64_t)-1;
654 c285a1f8 2020-11-06 stsp -	}
655 c285a1f8 2020-11-06 stsp +	sr_crypto_free_sessions(sd);
656 c285a1f8 2020-11-06 stsp  
657 c285a1f8 2020-11-06 stsp -	mtx_enter(&sd->mds.mdd_crypto.scr_mutex);
658 c285a1f8 2020-11-06 stsp -	while ((crwu = TAILQ_FIRST(&sd->mds.mdd_crypto.scr_wus)) != NULL) {
659 c285a1f8 2020-11-06 stsp -		TAILQ_REMOVE(&sd->mds.mdd_crypto.scr_wus, crwu, cr_link);
660 c285a1f8 2020-11-06 stsp -
661 c285a1f8 2020-11-06 stsp -		if (crwu->cr_dmabuf != NULL)
662 c285a1f8 2020-11-06 stsp +	TAILQ_FOREACH(wu, &sd->sd_wu, swu_next) {
663 c285a1f8 2020-11-06 stsp +		crwu = (struct sr_crypto_wu *)wu;
664 c285a1f8 2020-11-06 stsp +		if (crwu->cr_dmabuf)
665 c285a1f8 2020-11-06 stsp  			dma_free(crwu->cr_dmabuf, MAXPHYS);
666 c285a1f8 2020-11-06 stsp -		if (crwu->cr_crp) {
667 c285a1f8 2020-11-06 stsp -			/* twiddle cryptoreq back */
668 c285a1f8 2020-11-06 stsp -			crwu->cr_crp->crp_desc = crwu->cr_descs;
669 c285a1f8 2020-11-06 stsp +		if (crwu->cr_crp)
670 c285a1f8 2020-11-06 stsp  			crypto_freereq(crwu->cr_crp);
671 c285a1f8 2020-11-06 stsp -		}
672 c285a1f8 2020-11-06 stsp -		free(crwu, M_DEVBUF);
673 c285a1f8 2020-11-06 stsp  	}
674 c285a1f8 2020-11-06 stsp -	mtx_leave(&sd->mds.mdd_crypto.scr_mutex);
675 c285a1f8 2020-11-06 stsp  
676 c285a1f8 2020-11-06 stsp  	sr_wu_free(sd);
677 c285a1f8 2020-11-06 stsp  	sr_ccb_free(sd);
678 c285a1f8 2020-11-06 stsp @@ -1165,65 +1099,60 @@
679 c285a1f8 2020-11-06 stsp  sr_crypto_rw(struct sr_workunit *wu)
680 c285a1f8 2020-11-06 stsp  {
681 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu;
682 c285a1f8 2020-11-06 stsp -	int			s, rv = 0;
683 c285a1f8 2020-11-06 stsp +	daddr_t			blkno;
684 c285a1f8 2020-11-06 stsp +	int			rv = 0;
685 c285a1f8 2020-11-06 stsp  
686 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu: %p\n",
687 c285a1f8 2020-11-06 stsp +	DNPRINTF(SR_D_DIS, "%s: sr_crypto_rw wu %p\n",
688 c285a1f8 2020-11-06 stsp  	    DEVNAME(wu->swu_dis->sd_sc), wu);
689 c285a1f8 2020-11-06 stsp  
690 405e89a6 2022-08-02 op -	if (wu->swu_xs->flags & SCSI_DATA_OUT) {
691 c285a1f8 2020-11-06 stsp -		crwu = sr_crypto_wu_get(wu, 1);
692 c285a1f8 2020-11-06 stsp -		if (crwu == NULL)
693 c285a1f8 2020-11-06 stsp -			return (1);
694 405e89a6 2022-08-02 op +	if (sr_validate_io(wu, &blkno, "sr_crypto_rw"))
695 405e89a6 2022-08-02 op +		return (1);
696 405e89a6 2022-08-02 op +
697 405e89a6 2022-08-02 op +	if (wu->swu_xs->flags & SCSI_DATA_OUT) {
698 c285a1f8 2020-11-06 stsp +		crwu = sr_crypto_prepare(wu, 1);
699 c285a1f8 2020-11-06 stsp  		crwu->cr_crp->crp_callback = sr_crypto_write;
700 c285a1f8 2020-11-06 stsp -		s = splvm();
701 c285a1f8 2020-11-06 stsp -		if (crypto_invoke(crwu->cr_crp))
702 c285a1f8 2020-11-06 stsp -			rv = 1;
703 c285a1f8 2020-11-06 stsp -		else
704 c285a1f8 2020-11-06 stsp +		rv = crypto_dispatch(crwu->cr_crp);
705 c285a1f8 2020-11-06 stsp +		if (rv == 0)
706 c285a1f8 2020-11-06 stsp  			rv = crwu->cr_crp->crp_etype;
707 c285a1f8 2020-11-06 stsp -		splx(s);
708 c285a1f8 2020-11-06 stsp  	} else
709 c285a1f8 2020-11-06 stsp -		rv = sr_crypto_rw2(wu, NULL);
710 c285a1f8 2020-11-06 stsp +		rv = sr_crypto_dev_rw(wu, NULL);
711 c285a1f8 2020-11-06 stsp  
712 c285a1f8 2020-11-06 stsp  	return (rv);
713 c285a1f8 2020-11-06 stsp  }
714 c285a1f8 2020-11-06 stsp  
715 c285a1f8 2020-11-06 stsp -int
716 c285a1f8 2020-11-06 stsp +void
717 c285a1f8 2020-11-06 stsp  sr_crypto_write(struct cryptop *crp)
718 c285a1f8 2020-11-06 stsp  {
719 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu = crp->crp_opaque;
720 c285a1f8 2020-11-06 stsp -	struct sr_workunit	*wu = crwu->cr_wu;
721 c285a1f8 2020-11-06 stsp +	struct sr_workunit	*wu = &crwu->cr_wu;
722 c285a1f8 2020-11-06 stsp  	int			s;
723 c285a1f8 2020-11-06 stsp  
724 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %x xs: %x\n",
725 c285a1f8 2020-11-06 stsp +	DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %p xs: %p\n",
726 c285a1f8 2020-11-06 stsp  	    DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
727 c285a1f8 2020-11-06 stsp  
728 c285a1f8 2020-11-06 stsp  	if (crp->crp_etype) {
729 c285a1f8 2020-11-06 stsp  		/* fail io */
730 c285a1f8 2020-11-06 stsp  		wu->swu_xs->error = XS_DRIVER_STUFFUP;
731 c285a1f8 2020-11-06 stsp  		s = splbio();
732 c285a1f8 2020-11-06 stsp -		sr_crypto_finish_io(wu);
733 c285a1f8 2020-11-06 stsp +		sr_scsi_done(wu->swu_dis, wu->swu_xs);
734 c285a1f8 2020-11-06 stsp  		splx(s);
735 c285a1f8 2020-11-06 stsp  	}
736 c285a1f8 2020-11-06 stsp  
737 c285a1f8 2020-11-06 stsp -	return (sr_crypto_rw2(wu, crwu));
738 c285a1f8 2020-11-06 stsp +	sr_crypto_dev_rw(wu, crwu);
739 c285a1f8 2020-11-06 stsp  }
740 c285a1f8 2020-11-06 stsp  
741 c285a1f8 2020-11-06 stsp  int
742 c285a1f8 2020-11-06 stsp -sr_crypto_rw2(struct sr_workunit *wu, struct sr_crypto_wu *crwu)
743 c285a1f8 2020-11-06 stsp +sr_crypto_dev_rw(struct sr_workunit *wu, struct sr_crypto_wu *crwu)
744 c285a1f8 2020-11-06 stsp  {
745 c285a1f8 2020-11-06 stsp  	struct sr_discipline	*sd = wu->swu_dis;
746 c285a1f8 2020-11-06 stsp  	struct scsi_xfer	*xs = wu->swu_xs;
747 c285a1f8 2020-11-06 stsp  	struct sr_ccb		*ccb;
748 c285a1f8 2020-11-06 stsp  	struct uio		*uio;
749 c285a1f8 2020-11-06 stsp -	int			s;
750 c285a1f8 2020-11-06 stsp -	daddr64_t		blk;
751 c285a1f8 2020-11-06 stsp +	daddr_t			blkno;
752 c285a1f8 2020-11-06 stsp  
753 c285a1f8 2020-11-06 stsp -	if (sr_validate_io(wu, &blk, "sr_crypto_rw2"))
754 c285a1f8 2020-11-06 stsp -		goto bad;
755 c285a1f8 2020-11-06 stsp +	blkno = wu->swu_blk_start;
756 c285a1f8 2020-11-06 stsp  
757 c285a1f8 2020-11-06 stsp -	blk += sd->sd_meta->ssd_data_offset;
758 c285a1f8 2020-11-06 stsp -
759 c285a1f8 2020-11-06 stsp -	ccb = sr_ccb_rw(sd, 0, blk, xs->datalen, xs->data, xs->flags, 0);
760 c285a1f8 2020-11-06 stsp +	ccb = sr_ccb_rw(sd, 0, blkno, xs->datalen, xs->data, xs->flags, 0);
761 c285a1f8 2020-11-06 stsp  	if (!ccb) {
762 c285a1f8 2020-11-06 stsp  		/* should never happen but handle more gracefully */
763 c285a1f8 2020-11-06 stsp  		printf("%s: %s: too many ccbs queued\n",
764 c285a1f8 2020-11-06 stsp @@ -1236,17 +1165,10 @@
765 c285a1f8 2020-11-06 stsp  		ccb->ccb_opaque = crwu;
766 c285a1f8 2020-11-06 stsp  	}
767 c285a1f8 2020-11-06 stsp  	sr_wu_enqueue_ccb(wu, ccb);
768 c285a1f8 2020-11-06 stsp +	sr_schedule_wu(wu);
769 c285a1f8 2020-11-06 stsp  
770 c285a1f8 2020-11-06 stsp -	s = splbio();
771 c285a1f8 2020-11-06 stsp -
772 c285a1f8 2020-11-06 stsp -	if (sr_check_io_collision(wu))
773 c285a1f8 2020-11-06 stsp -		goto queued;
774 c285a1f8 2020-11-06 stsp -
775 c285a1f8 2020-11-06 stsp -	sr_raid_startwu(wu);
776 c285a1f8 2020-11-06 stsp -
777 c285a1f8 2020-11-06 stsp -queued:
778 c285a1f8 2020-11-06 stsp -	splx(s);
779 c285a1f8 2020-11-06 stsp  	return (0);
780 c285a1f8 2020-11-06 stsp +
781 c285a1f8 2020-11-06 stsp  bad:
782 c285a1f8 2020-11-06 stsp  	/* wu is unwound by sr_wu_put */
783 c285a1f8 2020-11-06 stsp  	if (crwu)
784 c285a1f8 2020-11-06 stsp @@ -1259,77 +1181,39 @@
785 c285a1f8 2020-11-06 stsp  {
786 c285a1f8 2020-11-06 stsp  	struct scsi_xfer	*xs = wu->swu_xs;
787 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu;
788 c285a1f8 2020-11-06 stsp -	struct sr_ccb		*ccb;
789 c285a1f8 2020-11-06 stsp  	int			s;
790 c285a1f8 2020-11-06 stsp  
791 c285a1f8 2020-11-06 stsp  	/* If this was a successful read, initiate decryption of the data. */
792 c285a1f8 2020-11-06 stsp  	if (ISSET(xs->flags, SCSI_DATA_IN) && xs->error == XS_NOERROR) {
793 c285a1f8 2020-11-06 stsp -		/* only fails on implementation error */
794 c285a1f8 2020-11-06 stsp -		crwu = sr_crypto_wu_get(wu, 0);
795 c285a1f8 2020-11-06 stsp -		if (crwu == NULL)
796 c285a1f8 2020-11-06 stsp -			panic("sr_crypto_intr: no wu");
797 c285a1f8 2020-11-06 stsp +		crwu = sr_crypto_prepare(wu, 0);
798 c285a1f8 2020-11-06 stsp  		crwu->cr_crp->crp_callback = sr_crypto_read;
799 c285a1f8 2020-11-06 stsp -		ccb = TAILQ_FIRST(&wu->swu_ccb);
800 c285a1f8 2020-11-06 stsp -		if (ccb == NULL)
801 c285a1f8 2020-11-06 stsp -			panic("sr_crypto_done: no ccbs on workunit");
802 c285a1f8 2020-11-06 stsp -		ccb->ccb_opaque = crwu;
803 c285a1f8 2020-11-06 stsp -		DNPRINTF(SR_D_INTR, "%s: sr_crypto_intr: crypto_invoke %p\n",
804 c285a1f8 2020-11-06 stsp +		DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_dispatch %p\n",
805 c285a1f8 2020-11-06 stsp  		    DEVNAME(wu->swu_dis->sd_sc), crwu->cr_crp);
806 c285a1f8 2020-11-06 stsp -		s = splvm();
807 c285a1f8 2020-11-06 stsp -		crypto_invoke(crwu->cr_crp);
808 c285a1f8 2020-11-06 stsp -		splx(s);
809 c285a1f8 2020-11-06 stsp +		crypto_dispatch(crwu->cr_crp);
810 c285a1f8 2020-11-06 stsp  		return;
811 c285a1f8 2020-11-06 stsp  	}
812 c285a1f8 2020-11-06 stsp  
813 c285a1f8 2020-11-06 stsp  	s = splbio();
814 c285a1f8 2020-11-06 stsp -	sr_crypto_finish_io(wu);
815 c285a1f8 2020-11-06 stsp +	sr_scsi_done(wu->swu_dis, wu->swu_xs);
816 c285a1f8 2020-11-06 stsp  	splx(s);
817 c285a1f8 2020-11-06 stsp  }
818 c285a1f8 2020-11-06 stsp  
819 c285a1f8 2020-11-06 stsp  void
820 c285a1f8 2020-11-06 stsp -sr_crypto_finish_io(struct sr_workunit *wu)
821 c285a1f8 2020-11-06 stsp -{
822 c285a1f8 2020-11-06 stsp -	struct sr_discipline	*sd = wu->swu_dis;
823 c285a1f8 2020-11-06 stsp -	struct scsi_xfer	*xs = wu->swu_xs;
824 c285a1f8 2020-11-06 stsp -	struct sr_ccb		*ccb;
825 c285a1f8 2020-11-06 stsp -#ifdef SR_DEBUG
826 c285a1f8 2020-11-06 stsp -	struct sr_softc		*sc = sd->sd_sc;
827 c285a1f8 2020-11-06 stsp -#endif /* SR_DEBUG */
828 c285a1f8 2020-11-06 stsp -
829 c285a1f8 2020-11-06 stsp -	splassert(IPL_BIO);
830 c285a1f8 2020-11-06 stsp -
831 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_finish_io: wu %x xs: %x\n",
832 c285a1f8 2020-11-06 stsp -	    DEVNAME(sc), wu, xs);
833 c285a1f8 2020-11-06 stsp -
834 c285a1f8 2020-11-06 stsp -	if (wu->swu_cb_active == 1)
835 c285a1f8 2020-11-06 stsp -		panic("%s: sr_crypto_finish_io", DEVNAME(sd->sd_sc));
836 c285a1f8 2020-11-06 stsp -	TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) {
837 c285a1f8 2020-11-06 stsp -		if (ccb->ccb_opaque == NULL)
838 c285a1f8 2020-11-06 stsp -			continue;
839 c285a1f8 2020-11-06 stsp -		sr_crypto_wu_put(ccb->ccb_opaque);
840 c285a1f8 2020-11-06 stsp -	}
841 c285a1f8 2020-11-06 stsp -
842 c285a1f8 2020-11-06 stsp -	sr_scsi_done(sd, xs);
843 c285a1f8 2020-11-06 stsp -}
844 c285a1f8 2020-11-06 stsp -
845 c285a1f8 2020-11-06 stsp -int
846 c285a1f8 2020-11-06 stsp  sr_crypto_read(struct cryptop *crp)
847 c285a1f8 2020-11-06 stsp  {
848 c285a1f8 2020-11-06 stsp  	struct sr_crypto_wu	*crwu = crp->crp_opaque;
849 c285a1f8 2020-11-06 stsp -	struct sr_workunit	*wu = crwu->cr_wu;
850 c285a1f8 2020-11-06 stsp +	struct sr_workunit	*wu = &crwu->cr_wu;
851 c285a1f8 2020-11-06 stsp  	int			s;
852 c285a1f8 2020-11-06 stsp  
853 c285a1f8 2020-11-06 stsp -	DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %x xs: %x\n",
854 c285a1f8 2020-11-06 stsp +	DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %p xs: %p\n",
855 c285a1f8 2020-11-06 stsp  	    DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
856 c285a1f8 2020-11-06 stsp  
857 c285a1f8 2020-11-06 stsp  	if (crp->crp_etype)
858 c285a1f8 2020-11-06 stsp  		wu->swu_xs->error = XS_DRIVER_STUFFUP;
859 c285a1f8 2020-11-06 stsp  
860 c285a1f8 2020-11-06 stsp  	s = splbio();
861 c285a1f8 2020-11-06 stsp -	sr_crypto_finish_io(wu);
862 c285a1f8 2020-11-06 stsp +	sr_scsi_done(wu->swu_dis, wu->swu_xs);
863 c285a1f8 2020-11-06 stsp  	splx(s);
864 c285a1f8 2020-11-06 stsp -
865 c285a1f8 2020-11-06 stsp -	return (0);
866 c285a1f8 2020-11-06 stsp  }
867 c285a1f8 2020-11-06 stsp  
868 c285a1f8 2020-11-06 stsp  void