Date: Mon, 18 Jan 2021 22:08:17 GMT From: Mark Johnston <markj@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 5bdb8b273aaf - main - safexcel: Maintain per-session context records Message-ID: <202101182208.10IM8HqH039266@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=5bdb8b273aaf6aa1bf089071ce6d5c9e80657c52 commit 5bdb8b273aaf6aa1bf089071ce6d5c9e80657c52 Author: Mark Johnston <markj@FreeBSD.org> AuthorDate: 2021-01-18 22:07:56 +0000 Commit: Mark Johnston <markj@FreeBSD.org> CommitDate: 2021-01-18 22:07:56 +0000 safexcel: Maintain per-session context records The context record contains key material precomputed by the driver at session creation time. Rather than storing various components of the context record in each session, go a bit further and store the full context record image so that safexcel_process() can simply copy the image into each request submitted to the hardware. This simplifies the data path and eliminates a bunch of unnecessary conditional logic that was getting executed for each request. MFC after: 1 week Sponsored by: Rubicon Communications, LLC (Netgate) --- sys/dev/safexcel/safexcel.c | 400 ++++++++++++++++++---------------------- sys/dev/safexcel/safexcel_var.h | 19 +- 2 files changed, 186 insertions(+), 233 deletions(-) diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c index 5f66cd9d4abb..3083f5f794fe 100644 --- a/sys/dev/safexcel/safexcel.c +++ b/sys/dev/safexcel/safexcel.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * - * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * Copyright (c) 2020, 2021 Rubicon Communications, LLC (Netgate) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -1280,59 +1280,153 @@ safexcel_detach(device_t dev) } /* - * Populate the request's context record with pre-computed key material. + * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted + * using the cipher key. + */ +static void +safexcel_setkey_ghash(const uint8_t *key, int klen, uint32_t *hashkey) +{ + uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; + uint8_t zeros[AES_BLOCK_LEN]; + int i, rounds; + + memset(zeros, 0, sizeof(zeros)); + + rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); + rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)hashkey); + for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) + hashkey[i] = htobe32(hashkey[i]); + + explicit_bzero(ks, sizeof(ks)); +} + +/* + * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 + * in the hardware implementation. K1 is the cipher key and comes last in the + * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC + * is not implemented so K2 and K3 are fixed. + */ +static void +safexcel_setkey_xcbcmac(const uint8_t *key, int klen, uint32_t *hashkey) +{ + int i, off; + + memset(hashkey, 0, 2 * AES_BLOCK_LEN); + off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); + for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) + hashkey[i + off] = htobe32(le32dec(key)); +} + +static void +safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, + char *buf) +{ + int hashwords, i; + + switch (ahash->type) { + case CRYPTO_SHA1_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); + break; + case CRYPTO_SHA2_224_HMAC: + hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); + break; + case CRYPTO_SHA2_256_HMAC: + hashwords = ahash->hashsize / sizeof(uint32_t); + for (i = 0; i < hashwords; i++) + ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); + break; + case CRYPTO_SHA2_384_HMAC: + hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); + break; + case CRYPTO_SHA2_512_HMAC: + hashwords = ahash->hashsize / sizeof(uint64_t); + for (i = 0; i < hashwords; i++) + ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); + break; + } +} + +/* + * Pre-compute the inner and outer digests used in the HMAC algorithm. + */ +static void +safexcel_setkey_hmac(const struct crypto_session_params *csp, + const uint8_t *key, int klen, uint8_t *ipad, uint8_t *opad) +{ + union authctx ctx; + struct auth_hash *ahash; + + ahash = crypto_auth_hash(csp); + hmac_init_ipad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, ipad); + hmac_init_opad(ahash, key, klen, &ctx); + safexcel_setkey_hmac_digest(ahash, &ctx, opad); + explicit_bzero(&ctx, ahash->ctxsize); +} + +static void +safexcel_setkey_xts(const uint8_t *key, int klen, uint8_t *tweakkey) +{ + memcpy(tweakkey, key + klen, klen); +} + +/* + * Populate a context record with paramters from a session. Some consumers + * specify per-request keys, in which case the context must be re-initialized + * for each request. */ static int -safexcel_set_context(struct safexcel_request *req) +safexcel_set_context(struct safexcel_context_record *ctx, int op, + const uint8_t *ckey, const uint8_t *akey, struct safexcel_session *sess) { const struct crypto_session_params *csp; - struct cryptop *crp; - struct safexcel_context_record *ctx; - struct safexcel_session *sess; uint8_t *data; - int off; + uint32_t ctrl0, ctrl1; + int aklen, alg, cklen, off; - crp = req->crp; - csp = crypto_get_params(crp->crp_session); - sess = req->sess; + csp = crypto_get_params(sess->cses); + aklen = csp->csp_auth_klen; + cklen = csp->csp_cipher_klen; + if (csp->csp_cipher_alg == CRYPTO_AES_XTS) + cklen /= 2; + + ctrl0 = sess->alg | sess->digest | sess->hash; + ctrl1 = sess->mode; - ctx = (struct safexcel_context_record *)req->ctx.vaddr; data = (uint8_t *)ctx->data; if (csp->csp_cipher_alg != 0) { - if (crp->crp_cipher_key != NULL) - memcpy(data, crp->crp_cipher_key, sess->klen); - else - memcpy(data, csp->csp_cipher_key, sess->klen); - off = sess->klen; + memcpy(data, ckey, cklen); + off = cklen; } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { - if (crp->crp_auth_key != NULL) - memcpy(data, crp->crp_auth_key, sess->klen); - else - memcpy(data, csp->csp_auth_key, sess->klen); - off = sess->klen; + memcpy(data, akey, aklen); + off = aklen; } else { off = 0; } switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: - memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + safexcel_setkey_ghash(ckey, cklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_AES_CCM_16: - memcpy(data + off, sess->xcbc_key, - AES_BLOCK_LEN * 2 + sess->klen); - off += AES_BLOCK_LEN * 2 + sess->klen; + safexcel_setkey_xcbcmac(ckey, cklen, (uint32_t *)(data + off)); + off += AES_BLOCK_LEN * 2 + cklen; break; case CRYPTO_AES_XTS: - memcpy(data + off, sess->tweak_key, sess->klen); - off += sess->klen; + safexcel_setkey_xts(ckey, cklen, data + off); + off += cklen; break; } - switch (csp->csp_auth_alg) { case CRYPTO_AES_NIST_GMAC: - memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + safexcel_setkey_ghash(akey, aklen, (uint32_t *)(data + off)); off += GMAC_BLOCK_LEN; break; case CRYPTO_SHA1_HMAC: @@ -1340,41 +1434,12 @@ safexcel_set_context(struct safexcel_request *req) case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: - memcpy(data + off, sess->hmac_ipad, sess->statelen); - off += sess->statelen; - memcpy(data + off, sess->hmac_opad, sess->statelen); - off += sess->statelen; + safexcel_setkey_hmac(csp, akey, aklen, + data + off, data + off + sess->statelen); + off += sess->statelen * 2; break; } - - return (off); -} - -/* - * Populate fields in the first command descriptor of the chain used to encode - * the specified request. These fields indicate the algorithms used, the size - * of the key material stored in the associated context record, the primitive - * operations to be performed on input data, and the location of the IV if any. - */ -static void -safexcel_set_command(struct safexcel_request *req, - struct safexcel_cmd_descr *cdesc) -{ - const struct crypto_session_params *csp; - struct cryptop *crp; - struct safexcel_session *sess; - uint32_t ctrl0, ctrl1, ctxr_len; - int alg; - - crp = req->crp; - csp = crypto_get_params(crp->crp_session); - sess = req->sess; - - ctrl0 = sess->alg | sess->digest | sess->hash; - ctrl1 = sess->mode; - - ctxr_len = safexcel_set_context(req) / sizeof(uint32_t); - ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len); + ctrl0 |= SAFEXCEL_CONTROL0_SIZE(off / sizeof(uint32_t)); alg = csp->csp_cipher_alg; if (alg == 0) @@ -1382,7 +1447,7 @@ safexcel_set_command(struct safexcel_request *req, switch (alg) { case CRYPTO_AES_CCM_16: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | SAFEXCEL_CONTROL0_KEY_EN; } else { @@ -1395,7 +1460,7 @@ safexcel_set_command(struct safexcel_request *req, case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + if (CRYPTO_OP_IS_ENCRYPT(op)) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN; if (csp->csp_auth_alg != 0) @@ -1410,8 +1475,7 @@ safexcel_set_command(struct safexcel_request *req, break; case CRYPTO_AES_NIST_GCM_16: case CRYPTO_AES_NIST_GMAC: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) || - csp->csp_auth_alg != 0) { + if (CRYPTO_OP_IS_ENCRYPT(op) || csp->csp_auth_alg != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_OUT; @@ -1442,8 +1506,10 @@ safexcel_set_command(struct safexcel_request *req, break; } - cdesc->control_data.control0 = ctrl0; - cdesc->control_data.control1 = ctrl1; + ctx->control0 = ctrl0; + ctx->control1 = ctrl1; + + return (off); } /* @@ -1809,17 +1875,48 @@ static void safexcel_set_token(struct safexcel_request *req) { const struct crypto_session_params *csp; + struct cryptop *crp; struct safexcel_cmd_descr *cdesc; + struct safexcel_context_record *ctx; + struct safexcel_context_template *ctxtmp; struct safexcel_instr *instr; struct safexcel_softc *sc; + const uint8_t *akey, *ckey; int ringidx; - csp = crypto_get_params(req->crp->crp_session); + crp = req->crp; + csp = crypto_get_params(crp->crp_session); cdesc = req->cdesc; sc = req->sc; ringidx = req->ringidx; - safexcel_set_command(req, cdesc); + akey = crp->crp_auth_key; + ckey = crp->crp_cipher_key; + if (akey != NULL || ckey != NULL) { + /* + * If we have a per-request key we have to generate the context + * record on the fly. + */ + if (akey == NULL) + akey = csp->csp_auth_key; + if (ckey == NULL) + ckey = csp->csp_cipher_key; + ctx = (struct safexcel_context_record *)req->ctx.vaddr; + (void)safexcel_set_context(ctx, crp->crp_op, ckey, akey, + req->sess); + } else { + /* + * Use the context record template computed at session + * initialization time. + */ + ctxtmp = CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ? + &req->sess->encctx : &req->sess->decctx; + ctx = &ctxtmp->ctx; + memcpy(req->ctx.vaddr + 2 * sizeof(uint32_t), ctx->data, + ctxtmp->len); + } + cdesc->control_data.control0 = ctx->control0; + cdesc->control_data.control1 = ctx->control1; /* * For keyless hash operations, the token instructions can be embedded @@ -2243,153 +2340,6 @@ safexcel_probesession(device_t dev, const struct crypto_session_params *csp) return (CRYPTODEV_PROBE_HARDWARE); } -/* - * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted - * using the cipher key. - */ -static void -safexcel_setkey_ghash(struct safexcel_session *sess, const uint8_t *key, - int klen) -{ - uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; - uint8_t zeros[AES_BLOCK_LEN]; - int i, rounds; - - memset(zeros, 0, sizeof(zeros)); - - rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); - rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)sess->ghash_key); - for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) - sess->ghash_key[i] = htobe32(sess->ghash_key[i]); - - explicit_bzero(ks, sizeof(ks)); -} - -/* - * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 - * in the hardware implementation. K1 is the cipher key and comes last in the - * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC - * is not implemented so K2 and K3 are fixed. - */ -static void -safexcel_setkey_xcbcmac(struct safexcel_session *sess, const uint8_t *key, - int klen) -{ - int i, off; - - memset(sess->xcbc_key, 0, sizeof(sess->xcbc_key)); - off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); - for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) - sess->xcbc_key[i + off] = htobe32(le32dec(key)); -} - -static void -safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, - char *buf) -{ - int hashwords, i; - - switch (ahash->type) { - case CRYPTO_SHA1_HMAC: - hashwords = ahash->hashsize / sizeof(uint32_t); - for (i = 0; i < hashwords; i++) - ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); - break; - case CRYPTO_SHA2_224_HMAC: - hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); - for (i = 0; i < hashwords; i++) - ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); - break; - case CRYPTO_SHA2_256_HMAC: - hashwords = ahash->hashsize / sizeof(uint32_t); - for (i = 0; i < hashwords; i++) - ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); - break; - case CRYPTO_SHA2_384_HMAC: - hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); - for (i = 0; i < hashwords; i++) - ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); - break; - case CRYPTO_SHA2_512_HMAC: - hashwords = ahash->hashsize / sizeof(uint64_t); - for (i = 0; i < hashwords; i++) - ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); - break; - } -} - -/* - * Pre-compute the inner and outer digests used in the HMAC algorithm. - */ -static void -safexcel_setkey_hmac(const struct crypto_session_params *csp, - struct safexcel_session *sess, const uint8_t *key, int klen) -{ - union authctx ctx; - struct auth_hash *ahash; - - ahash = crypto_auth_hash(csp); - hmac_init_ipad(ahash, key, klen, &ctx); - safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_ipad); - hmac_init_opad(ahash, key, klen, &ctx); - safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_opad); - explicit_bzero(&ctx, ahash->ctxsize); -} - -static void -safexcel_setkey_xts(struct safexcel_session *sess, const uint8_t *key, int klen) -{ - memcpy(sess->tweak_key, key + klen / 2, klen / 2); -} - -static void -safexcel_setkey(struct safexcel_session *sess, - const struct crypto_session_params *csp, struct cryptop *crp) -{ - const uint8_t *akey, *ckey; - int aklen, cklen; - - aklen = csp->csp_auth_klen; - cklen = csp->csp_cipher_klen; - akey = ckey = NULL; - if (crp != NULL) { - akey = crp->crp_auth_key; - ckey = crp->crp_cipher_key; - } - if (akey == NULL) - akey = csp->csp_auth_key; - if (ckey == NULL) - ckey = csp->csp_cipher_key; - - sess->klen = cklen; - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_NIST_GCM_16: - safexcel_setkey_ghash(sess, ckey, cklen); - break; - case CRYPTO_AES_CCM_16: - safexcel_setkey_xcbcmac(sess, ckey, cklen); - break; - case CRYPTO_AES_XTS: - safexcel_setkey_xts(sess, ckey, cklen); - sess->klen /= 2; - break; - } - - switch (csp->csp_auth_alg) { - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - safexcel_setkey_hmac(csp, sess, akey, aklen); - break; - case CRYPTO_AES_NIST_GMAC: - sess->klen = aklen; - safexcel_setkey_ghash(sess, akey, aklen); - break; - } -} - static uint32_t safexcel_aes_algid(int keylen) { @@ -2499,6 +2449,7 @@ safexcel_newsession(device_t dev, crypto_session_t cses, sc = device_get_softc(dev); sess = crypto_get_driver_session(cses); + sess->cses = cses; switch (csp->csp_auth_alg) { case CRYPTO_SHA1: @@ -2562,7 +2513,15 @@ safexcel_newsession(device_t dev, crypto_session_t cses, if (csp->csp_auth_mlen != 0) sess->digestlen = csp->csp_auth_mlen; - safexcel_setkey(sess, csp, NULL); + if ((csp->csp_cipher_alg == 0 || csp->csp_cipher_key != NULL) && + (csp->csp_auth_alg == 0 || csp->csp_auth_key != NULL)) { + sess->encctx.len = safexcel_set_context(&sess->encctx.ctx, + CRYPTO_OP_ENCRYPT, csp->csp_cipher_key, csp->csp_auth_key, + sess); + sess->decctx.len = safexcel_set_context(&sess->decctx.ctx, + CRYPTO_OP_DECRYPT, csp->csp_cipher_key, csp->csp_auth_key, + sess); + } return (0); } @@ -2588,13 +2547,10 @@ safexcel_process(device_t dev, struct cryptop *crp, int hint) return (0); } - if (crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL) - safexcel_setkey(sess, csp, crp); - ring = &sc->sc_ring[curcpu % sc->sc_config.rings]; mtx_lock(&ring->mtx); req = safexcel_alloc_request(sc, ring); - if (__predict_false(req == NULL)) { + if (__predict_false(req == NULL)) { ring->blocked = CRYPTO_SYMQ; mtx_unlock(&ring->mtx); counter_u64_add(sc->sc_req_alloc_failures, 1); diff --git a/sys/dev/safexcel/safexcel_var.h b/sys/dev/safexcel/safexcel_var.h index 5a81c3086492..efcf66a090bf 100644 --- a/sys/dev/safexcel/safexcel_var.h +++ b/sys/dev/safexcel/safexcel_var.h @@ -335,24 +335,21 @@ struct safexcel_res_descr_ring { int read; }; +struct safexcel_context_template { + struct safexcel_context_record ctx; + int len; +}; + struct safexcel_session { + crypto_session_t cses; uint32_t alg; /* cipher algorithm */ uint32_t digest; /* digest type */ uint32_t hash; /* hash algorithm */ uint32_t mode; /* cipher mode of operation */ unsigned int digestlen; /* digest length */ unsigned int statelen; /* HMAC hash state length */ - unsigned int klen; /* cipher key length */ - union { - uint32_t ghash_key[AES_BLOCK_LEN / sizeof(uint32_t)]; - uint32_t xcbc_key[(AES_BLOCK_LEN * 2 + AES_MAX_KEY) / - sizeof(uint32_t)]; - uint8_t tweak_key[AES_MAX_KEY]; - }; - struct { - uint8_t hmac_ipad[HMAC_MAX_BLOCK_LEN]; - uint8_t hmac_opad[HMAC_MAX_BLOCK_LEN]; - }; + + struct safexcel_context_template encctx, decctx; }; struct safexcel_softc;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202101182208.10IM8HqH039266>