algif_aead.c 16 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * algif_aead: User-space interface for AEAD algorithms
 *
 * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
 *
 * This file provides the user-space API for AEAD ciphers.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * The following concept of the memory management is used:
 *
 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
 * filled by user space with the data submitted via sendpage/sendmsg. Filling
 * up the TX SGL does not cause a crypto operation -- the data will only be
 * tracked by the kernel. Upon receipt of one recvmsg call, the caller must
 * provide a buffer which is tracked with the RX SGL.
 *
 * During the processing of the recvmsg operation, the cipher request is
 * allocated and prepared. As part of the recvmsg operation, the processed
 * TX buffers are extracted from the TX SGL into a separate SGL.
 *
 * After the completion of the crypto operation, the RX SGL and the cipher
 * request is released. The extracted TX SGL parts are released together with
 * the RX SGL release.
28 29
 */

30
#include <crypto/internal/aead.h>
31 32
#include <crypto/scatterwalk.h>
#include <crypto/if_alg.h>
33 34
#include <crypto/skcipher.h>
#include <crypto/null.h>
35 36 37 38 39 40 41 42
#include <linux/init.h>
#include <linux/list.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/net.h>
#include <net/sock.h>

43 44
struct aead_tfm {
	struct crypto_aead *aead;
45
	struct crypto_skcipher *null_tfm;
46 47
};

48 49 50 51 52
static inline bool aead_sufficient_data(struct sock *sk)
{
	struct alg_sock *ask = alg_sk(sk);
	struct sock *psk = ask->parent;
	struct alg_sock *pask = alg_sk(psk);
53
	struct af_alg_ctx *ctx = ask->private;
54 55 56
	struct aead_tfm *aeadc = pask->private;
	struct crypto_aead *tfm = aeadc->aead;
	unsigned int as = crypto_aead_authsize(tfm);
57

58 59 60 61 62
	/*
	 * The minimum amount of memory needed for an AEAD cipher is
	 * the AAD and in case of decryption the tag.
	 */
	return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
63 64
}

65
static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
66 67 68
{
	struct sock *sk = sock->sk;
	struct alg_sock *ask = alg_sk(sk);
69 70 71 72 73
	struct sock *psk = ask->parent;
	struct alg_sock *pask = alg_sk(psk);
	struct aead_tfm *aeadc = pask->private;
	struct crypto_aead *tfm = aeadc->aead;
	unsigned int ivsize = crypto_aead_ivsize(tfm);
74

75
	return af_alg_sendmsg(sock, msg, size, ivsize);
76 77
}

78 79 80 81 82 83 84 85 86 87 88 89 90 91
static int crypto_aead_copy_sgl(struct crypto_skcipher *null_tfm,
				struct scatterlist *src,
				struct scatterlist *dst, unsigned int len)
{
	SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);

	skcipher_request_set_tfm(skreq, null_tfm);
	skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG,
				      NULL, NULL);
	skcipher_request_set_crypt(skreq, src, dst, len, NULL);

	return crypto_skcipher_encrypt(skreq);
}

92 93
static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
			 size_t ignored, int flags)
94 95 96
{
	struct sock *sk = sock->sk;
	struct alg_sock *ask = alg_sk(sk);
97 98
	struct sock *psk = ask->parent;
	struct alg_sock *pask = alg_sk(psk);
99
	struct af_alg_ctx *ctx = ask->private;
100 101
	struct aead_tfm *aeadc = pask->private;
	struct crypto_aead *tfm = aeadc->aead;
102
	struct crypto_skcipher *null_tfm = aeadc->null_tfm;
103
	unsigned int i, as = crypto_aead_authsize(tfm);
104
	struct af_alg_async_req *areq;
105 106
	struct af_alg_tsgl *tsgl, *tmp;
	struct scatterlist *rsgl_src, *tsgl_src = NULL;
107 108 109 110 111
	int err = 0;
	size_t used = 0;		/* [in]  TX bufs to be en/decrypted */
	size_t outlen = 0;		/* [out] RX bufs produced by kernel */
	size_t usedpages = 0;		/* [in]  RX bufs to be used from user */
	size_t processed = 0;		/* [in]  TX bufs to be consumed */
112

113 114 115 116 117 118
	if (!ctx->used) {
		err = af_alg_wait_for_data(sk, flags);
		if (err)
			return err;
	}

119
	/*
120 121
	 * Data length provided by caller via sendmsg/sendpage that has not
	 * yet been processed.
122 123 124 125 126 127 128 129 130 131 132 133
	 */
	used = ctx->used;

	/*
	 * Make sure sufficient data is present -- note, the same check is
	 * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
	 * shall provide an information to the data sender that something is
	 * wrong, but they are irrelevant to maintain the kernel integrity.
	 * We need this check here too in case user space decides to not honor
	 * the error message in sendmsg/sendpage and still call recvmsg. This
	 * check here protects the kernel integrity.
	 */
134 135
	if (!aead_sufficient_data(sk))
		return -EINVAL;
136

137 138 139 140 141 142 143 144 145 146 147 148
	/*
	 * Calculate the minimum output buffer size holding the result of the
	 * cipher operation. When encrypting data, the receiving buffer is
	 * larger by the tag length compared to the input buffer as the
	 * encryption operation generates the tag. For decryption, the input
	 * buffer provides the tag which is consumed resulting in only the
	 * plaintext without a buffer for the tag returned to the caller.
	 */
	if (ctx->enc)
		outlen = used + as;
	else
		outlen = used - as;
149

150 151 152 153
	/*
	 * The cipher operation input data is reduced by the associated data
	 * length as this data is processed separately later on.
	 */
154
	used -= ctx->aead_assoclen;
155

156
	/* Allocate cipher request for current operation. */
157 158 159 160
	areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
				     crypto_aead_reqsize(tfm));
	if (IS_ERR(areq))
		return PTR_ERR(areq);
161 162

	/* convert iovecs of output buffers into RX SGL */
163 164 165
	err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
	if (err)
		goto free;
166

167 168 169 170 171 172 173
	/*
	 * Ensure output buffer is sufficiently large. If the caller provides
	 * less buffer space, only use the relative required input size. This
	 * allows AIO operation where the caller sent all data to be processed
	 * and the AIO operation performs the operation on the different chunks
	 * of the input data.
	 */
174
	if (usedpages < outlen) {
175
		size_t less = outlen - usedpages;
176

177 178 179 180 181 182 183
		if (used < less) {
			err = -EINVAL;
			goto free;
		}
		used -= less;
		outlen -= less;
	}
184

185
	processed = used + ctx->aead_assoclen;
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
	list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) {
		for (i = 0; i < tsgl->cur; i++) {
			struct scatterlist *process_sg = tsgl->sg + i;

			if (!(process_sg->length) || !sg_page(process_sg))
				continue;
			tsgl_src = process_sg;
			break;
		}
		if (tsgl_src)
			break;
	}
	if (processed && !tsgl_src) {
		err = -EFAULT;
		goto free;
	}
202

203
	/*
204 205 206 207 208 209 210 211 212 213
	 * Copy of AAD from source to destination
	 *
	 * The AAD is copied to the destination buffer without change. Even
	 * when user space uses an in-place cipher operation, the kernel
	 * will copy the data as it does not see whether such in-place operation
	 * is initiated.
	 *
	 * To ensure efficiency, the following implementation ensure that the
	 * ciphers are invoked to perform a crypto operation in-place. This
	 * is achieved by memory management specified as follows.
214
	 */
215 216

	/* Use the RX SGL as source (and destination) for crypto op. */
217
	rsgl_src = areq->first_rsgl.sgl.sg;
218 219 220 221 222 223

	if (ctx->enc) {
		/*
		 * Encryption operation - The in-place cipher operation is
		 * achieved by the following operation:
		 *
224
		 * TX SGL: AAD || PT
225 226 227
		 *	    |	   |
		 *	    | copy |
		 *	    v	   v
228
		 * RX SGL: AAD || PT || Tag
229
		 */
230
		err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
231 232 233
					   areq->first_rsgl.sgl.sg, processed);
		if (err)
			goto free;
234
		af_alg_pull_tsgl(sk, processed, NULL, 0);
235 236 237 238 239 240 241 242 243 244 245 246 247
	} else {
		/*
		 * Decryption operation - To achieve an in-place cipher
		 * operation, the following  SGL structure is used:
		 *
		 * TX SGL: AAD || CT || Tag
		 *	    |	   |	 ^
		 *	    | copy |	 | Create SGL link.
		 *	    v	   v	 |
		 * RX SGL: AAD || CT ----+
		 */

		 /* Copy AAD || CT to RX SGL buffer for in-place operation. */
248
		err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
249 250 251 252 253
					   areq->first_rsgl.sgl.sg, outlen);
		if (err)
			goto free;

		/* Create TX SGL for tag and chain it to RX SGL. */
254 255
		areq->tsgl_entries = af_alg_count_tsgl(sk, processed,
						       processed - as);
256 257 258 259 260 261 262 263 264 265 266 267
		if (!areq->tsgl_entries)
			areq->tsgl_entries = 1;
		areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) *
					      areq->tsgl_entries,
					  GFP_KERNEL);
		if (!areq->tsgl) {
			err = -ENOMEM;
			goto free;
		}
		sg_init_table(areq->tsgl, areq->tsgl_entries);

		/* Release TX SGL, except for tag data and reassign tag data. */
268
		af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);
269 270

		/* chain the areq TX SGL holding the tag with RX SGL */
271
		if (usedpages) {
272
			/* RX SGL present */
273
			struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;
274 275 276 277 278 279

			sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
			sg_chain(sgl_prev->sg, sgl_prev->npages + 1,
				 areq->tsgl);
		} else
			/* no RX SGL present (e.g. authentication only) */
280
			rsgl_src = areq->tsgl;
281 282 283
	}

	/* Initialize the crypto operation */
284
	aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src,
285
			       areq->first_rsgl.sgl.sg, used, ctx->iv);
286 287
	aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
	aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
288 289 290

	if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
		/* AIO operation */
291
		sock_hold(sk);
292
		areq->iocb = msg->msg_iocb;
293 294 295 296

		/* Remember output size that will be generated. */
		areq->outlen = outlen;

297
		aead_request_set_callback(&areq->cra_u.aead_req,
298
					  CRYPTO_TFM_REQ_MAY_BACKLOG,
299 300 301
					  af_alg_async_cb, areq);
		err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
				 crypto_aead_decrypt(&areq->cra_u.aead_req);
302 303

		/* AIO operation in progress */
304
		if (err == -EINPROGRESS || err == -EBUSY)
305 306 307
			return -EIOCBQUEUED;

		sock_put(sk);
308 309
	} else {
		/* Synchronous operation */
310
		aead_request_set_callback(&areq->cra_u.aead_req,
311
					  CRYPTO_TFM_REQ_MAY_BACKLOG,
312 313
					  crypto_req_done, &ctx->wait);
		err = crypto_wait_req(ctx->enc ?
314 315
				crypto_aead_encrypt(&areq->cra_u.aead_req) :
				crypto_aead_decrypt(&areq->cra_u.aead_req),
316
				&ctx->wait);
317 318
	}

319 320

free:
321
	af_alg_free_resources(areq);
322 323 324 325

	return err ? err : outlen;
}

326 327
static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
			size_t ignored, int flags)
328
{
329 330 331 332 333 334 335 336 337 338 339 340
	struct sock *sk = sock->sk;
	int ret = 0;

	lock_sock(sk);
	while (msg_data_left(msg)) {
		int err = _aead_recvmsg(sock, msg, ignored, flags);

		/*
		 * This error covers -EIOCBQUEUED which implies that we can
		 * only handle one AIO request. If the caller wants to have
		 * multiple AIO requests in parallel, he must make multiple
		 * separate AIO calls.
341 342
		 *
		 * Also return the error if no data has been processed so far.
343 344
		 */
		if (err <= 0) {
345
			if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
346 347 348 349 350 351 352 353
				ret = err;
			goto out;
		}

		ret += err;
	}

out:
354
	af_alg_wmem_wakeup(sk);
355 356
	release_sock(sk);
	return ret;
357 358
}

359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
static struct proto_ops algif_aead_ops = {
	.family		=	PF_ALG,

	.connect	=	sock_no_connect,
	.socketpair	=	sock_no_socketpair,
	.getname	=	sock_no_getname,
	.ioctl		=	sock_no_ioctl,
	.listen		=	sock_no_listen,
	.shutdown	=	sock_no_shutdown,
	.getsockopt	=	sock_no_getsockopt,
	.mmap		=	sock_no_mmap,
	.bind		=	sock_no_bind,
	.accept		=	sock_no_accept,
	.setsockopt	=	sock_no_setsockopt,

	.release	=	af_alg_release,
	.sendmsg	=	aead_sendmsg,
376
	.sendpage	=	af_alg_sendpage,
377
	.recvmsg	=	aead_recvmsg,
378
	.poll		=	af_alg_poll,
379 380
};

381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
static int aead_check_key(struct socket *sock)
{
	int err = 0;
	struct sock *psk;
	struct alg_sock *pask;
	struct aead_tfm *tfm;
	struct sock *sk = sock->sk;
	struct alg_sock *ask = alg_sk(sk);

	lock_sock(sk);
	if (ask->refcnt)
		goto unlock_child;

	psk = ask->parent;
	pask = alg_sk(ask->parent);
	tfm = pask->private;

	err = -ENOKEY;
	lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
400
	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
		goto unlock;

	if (!pask->refcnt++)
		sock_hold(psk);

	ask->refcnt = 1;
	sock_put(psk);

	err = 0;

unlock:
	release_sock(psk);
unlock_child:
	release_sock(sk);

	return err;
}

static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
				  size_t size)
{
	int err;

	err = aead_check_key(sock);
	if (err)
		return err;

	return aead_sendmsg(sock, msg, size);
}

static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page,
				       int offset, size_t size, int flags)
{
	int err;

	err = aead_check_key(sock);
	if (err)
		return err;

440
	return af_alg_sendpage(sock, page, offset, size, flags);
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
}

static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
				  size_t ignored, int flags)
{
	int err;

	err = aead_check_key(sock);
	if (err)
		return err;

	return aead_recvmsg(sock, msg, ignored, flags);
}

static struct proto_ops algif_aead_ops_nokey = {
	.family		=	PF_ALG,

	.connect	=	sock_no_connect,
	.socketpair	=	sock_no_socketpair,
	.getname	=	sock_no_getname,
	.ioctl		=	sock_no_ioctl,
	.listen		=	sock_no_listen,
	.shutdown	=	sock_no_shutdown,
	.getsockopt	=	sock_no_getsockopt,
	.mmap		=	sock_no_mmap,
	.bind		=	sock_no_bind,
	.accept		=	sock_no_accept,
	.setsockopt	=	sock_no_setsockopt,

	.release	=	af_alg_release,
	.sendmsg	=	aead_sendmsg_nokey,
	.sendpage	=	aead_sendpage_nokey,
	.recvmsg	=	aead_recvmsg_nokey,
474
	.poll		=	af_alg_poll,
475 476
};

477 478
static void *aead_bind(const char *name, u32 type, u32 mask)
{
479 480
	struct aead_tfm *tfm;
	struct crypto_aead *aead;
481
	struct crypto_skcipher *null_tfm;
482 483 484 485 486 487 488 489 490 491 492

	tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
	if (!tfm)
		return ERR_PTR(-ENOMEM);

	aead = crypto_alloc_aead(name, type, mask);
	if (IS_ERR(aead)) {
		kfree(tfm);
		return ERR_CAST(aead);
	}

493
	null_tfm = crypto_get_default_null_skcipher();
494 495 496 497 498 499
	if (IS_ERR(null_tfm)) {
		crypto_free_aead(aead);
		kfree(tfm);
		return ERR_CAST(null_tfm);
	}

500
	tfm->aead = aead;
501
	tfm->null_tfm = null_tfm;
502 503

	return tfm;
504 505 506 507
}

static void aead_release(void *private)
{
508 509 510
	struct aead_tfm *tfm = private;

	crypto_free_aead(tfm->aead);
511
	crypto_put_default_null_skcipher();
512
	kfree(tfm);
513 514 515 516
}

static int aead_setauthsize(void *private, unsigned int authsize)
{
517 518 519
	struct aead_tfm *tfm = private;

	return crypto_aead_setauthsize(tfm->aead, authsize);
520 521 522 523
}

static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
{
524 525
	struct aead_tfm *tfm = private;

526
	return crypto_aead_setkey(tfm->aead, key, keylen);
527 528 529 530 531
}

static void aead_sock_destruct(struct sock *sk)
{
	struct alg_sock *ask = alg_sk(sk);
532
	struct af_alg_ctx *ctx = ask->private;
533 534 535 536 537
	struct sock *psk = ask->parent;
	struct alg_sock *pask = alg_sk(psk);
	struct aead_tfm *aeadc = pask->private;
	struct crypto_aead *tfm = aeadc->aead;
	unsigned int ivlen = crypto_aead_ivsize(tfm);
538

539
	af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
540 541 542 543 544
	sock_kzfree_s(sk, ctx->iv, ivlen);
	sock_kfree_s(sk, ctx, ctx->len);
	af_alg_release_parent(sk);
}

545
static int aead_accept_parent_nokey(void *private, struct sock *sk)
546
{
547
	struct af_alg_ctx *ctx;
548
	struct alg_sock *ask = alg_sk(sk);
549 550
	struct aead_tfm *tfm = private;
	struct crypto_aead *aead = tfm->aead;
551
	unsigned int len = sizeof(*ctx);
552
	unsigned int ivlen = crypto_aead_ivsize(aead);
553 554 555 556 557 558 559 560 561 562 563 564 565

	ctx = sock_kmalloc(sk, len, GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;
	memset(ctx, 0, len);

	ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
	if (!ctx->iv) {
		sock_kfree_s(sk, ctx, len);
		return -ENOMEM;
	}
	memset(ctx->iv, 0, ivlen);

566
	INIT_LIST_HEAD(&ctx->tsgl_list);
567 568
	ctx->len = len;
	ctx->used = 0;
569
	atomic_set(&ctx->rcvused, 0);
570 571 572 573
	ctx->more = 0;
	ctx->merge = 0;
	ctx->enc = 0;
	ctx->aead_assoclen = 0;
574
	crypto_init_wait(&ctx->wait);
575 576 577 578 579 580 581 582

	ask->private = ctx;

	sk->sk_destruct = aead_sock_destruct;

	return 0;
}

583 584 585 586
static int aead_accept_parent(void *private, struct sock *sk)
{
	struct aead_tfm *tfm = private;

587
	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
588 589 590 591 592
		return -ENOKEY;

	return aead_accept_parent_nokey(private, sk);
}

593 594 595 596 597 598
static const struct af_alg_type algif_type_aead = {
	.bind		=	aead_bind,
	.release	=	aead_release,
	.setkey		=	aead_setkey,
	.setauthsize	=	aead_setauthsize,
	.accept		=	aead_accept_parent,
599
	.accept_nokey	=	aead_accept_parent_nokey,
600
	.ops		=	&algif_aead_ops,
601
	.ops_nokey	=	&algif_aead_ops_nokey,
602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
	.name		=	"aead",
	.owner		=	THIS_MODULE
};

static int __init algif_aead_init(void)
{
	return af_alg_register_type(&algif_type_aead);
}

static void __exit algif_aead_exit(void)
{
	int err = af_alg_unregister_type(&algif_type_aead);
	BUG_ON(err);
}

module_init(algif_aead_init);
module_exit(algif_aead_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");