Date: Wed, 22 Jul 2020 14:22:35 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r363418 - in stable/12: share/man/man4 sys/arm64/conf sys/conf sys/dev/safexcel sys/modules sys/modules/safexcel Message-ID: <202007221422.06MEMZUK013536@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Wed Jul 22 14:22:35 2020 New Revision: 363418 URL: https://svnweb.freebsd.org/changeset/base/363418 Log: MFC r363180, r363182, r363251: Add a driver for the SafeXcel EIP-97. Added: stable/12/share/man/man4/safexcel.4 - copied unchanged from r363180, head/share/man/man4/safexcel.4 stable/12/sys/dev/safexcel/ - copied from r363180, head/sys/dev/safexcel/ stable/12/sys/modules/safexcel/ - copied from r363180, head/sys/modules/safexcel/ Modified: stable/12/share/man/man4/Makefile stable/12/sys/arm64/conf/GENERIC stable/12/sys/conf/files.arm64 stable/12/sys/dev/safexcel/safexcel.c stable/12/sys/dev/safexcel/safexcel_var.h stable/12/sys/modules/Makefile Directory Properties: stable/12/ (props changed) Modified: stable/12/share/man/man4/Makefile ============================================================================== --- stable/12/share/man/man4/Makefile Wed Jul 22 13:49:54 2020 (r363417) +++ stable/12/share/man/man4/Makefile Wed Jul 22 14:22:35 2020 (r363418) @@ -461,6 +461,7 @@ MAN= aac.4 \ rue.4 \ sa.4 \ safe.4 \ + safexcel.4 \ sbp.4 \ sbp_targ.4 \ scc.4 \ Copied: stable/12/share/man/man4/safexcel.4 (from r363180, head/share/man/man4/safexcel.4) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/12/share/man/man4/safexcel.4 Wed Jul 22 14:22:35 2020 (r363418, copy of r363180, head/share/man/man4/safexcel.4) @@ -0,0 +1,84 @@ +.\"- +.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate) +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd June 23, 2020 +.Dt SAFEXCEL 4 +.Os +.Sh NAME +.Nm safexcel +.Nd Inside Secure SafeXcel-IP-97 security packet engine +.Sh SYNOPSIS +To compile this driver into the kernel, +place the following lines in your +kernel configuration file: +.Bd -ragged -offset indent +.Cd "device crypto" +.Cd "device cryptodev" +.Cd "device safexcel" +.Ed +.Pp +Alternatively, to load the driver as a +module at boot time, place the following line in +.Xr loader.conf 5 : +.Bd -literal -offset indent +safexcel_load="YES" +.Ed +.Sh DESCRIPTION +The +.Nm +driver implements +.Xr crypto 4 +support for the cryptographic acceleration functions of the EIP-97 device +found on some Marvell systems-on-chip. +The driver can accelerate the following AES modes: +.Pp +.Bl -bullet -compact +.It +AES-CBC +.It +AES-CTR +.It +AES-XTS +.It +AES-GCM +.It +AES-CCM +.El +.Pp +.Nm +also implements SHA1 and SHA2 transforms, and can combine AES-CBC and AES-CTR +with SHA1-HMAC and SHA2-HMAC for encrypt-then-authenticate operations. +.Sh SEE ALSO +.Xr crypto 4 , +.Xr ipsec 4 , +.Xr random 4 , +.Xr geli 8 , +.Xr crypto 9 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 13.0 . Modified: stable/12/sys/arm64/conf/GENERIC ============================================================================== --- stable/12/sys/arm64/conf/GENERIC Wed Jul 22 13:49:54 2020 (r363417) +++ stable/12/sys/arm64/conf/GENERIC Wed Jul 22 14:22:35 2020 (r363418) @@ -238,6 +238,9 @@ device mv_ap806_gicp # Marvell AP806 GICP device aw_rtc # Allwinner Real-time Clock device mv_rtc # Marvell Real-time Clock +# Crypto accelerators +device safexcel # Inside Secure EIP-97 + # Watchdog controllers device aw_wdog # Allwinner Watchdog Modified: stable/12/sys/conf/files.arm64 ============================================================================== --- stable/12/sys/conf/files.arm64 Wed Jul 22 13:49:54 2020 (r363417) +++ stable/12/sys/conf/files.arm64 Wed Jul 22 14:22:35 2020 (r363418) @@ -240,6 +240,7 @@ dev/pci/pci_host_generic_fdt.c optional pci fdt dev/psci/psci.c standard dev/psci/psci_arm64.S standard dev/psci/smccc.c standard +dev/safexcel/safexcel.c optional safexcel fdt dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci fdt dev/uart/uart_cpu_arm64.c optional uart dev/uart/uart_dev_mu.c optional uart uart_mu Modified: stable/12/sys/dev/safexcel/safexcel.c ============================================================================== --- head/sys/dev/safexcel/safexcel.c Tue Jul 14 14:09:29 2020 (r363180) +++ stable/12/sys/dev/safexcel/safexcel.c Wed Jul 22 14:22:35 2020 (r363418) @@ -901,7 +901,8 @@ static int safexcel_dma_init(struct safexcel_softc *sc) { struct safexcel_ring *ring; - int error, i, size; + bus_size_t size; + int error, i; for (i = 0; i < sc->sc_config.rings; i++) { ring = &sc->sc_ring[i]; @@ -937,8 +938,9 @@ safexcel_dma_init(struct safexcel_softc *sc) (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; /* Allocate additional CDR token memory. */ - error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, - sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE); + size = (bus_size_t)sc->sc_config.atok_offset * + SAFEXCEL_RING_SIZE; + error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size); if (error != 0) { device_printf(sc->sc_dev, "failed to allocate atoken DMA memory, error %d\n", @@ -1144,6 +1146,12 @@ safexcel_probe(device_t dev) return (BUS_PROBE_DEFAULT); } +static void +safexcel_crypto_register(struct safexcel_softc *sc, int alg) +{ + (void)crypto_register(sc->sc_cid, alg, SAFEXCEL_MAX_REQUEST_SIZE, 0); +} + static int safexcel_attach(device_t dev) { @@ -1211,6 +1219,26 @@ safexcel_attach(device_t dev) if (sc->sc_cid < 0) goto err2; + safexcel_crypto_register(sc, CRYPTO_AES_CBC); + safexcel_crypto_register(sc, CRYPTO_AES_ICM); + safexcel_crypto_register(sc, CRYPTO_AES_XTS); + safexcel_crypto_register(sc, CRYPTO_AES_CCM_16); + safexcel_crypto_register(sc, CRYPTO_AES_CCM_CBC_MAC); + safexcel_crypto_register(sc, CRYPTO_AES_NIST_GCM_16); + safexcel_crypto_register(sc, CRYPTO_AES_128_NIST_GMAC); + safexcel_crypto_register(sc, CRYPTO_AES_192_NIST_GMAC); + safexcel_crypto_register(sc, CRYPTO_AES_256_NIST_GMAC); + safexcel_crypto_register(sc, CRYPTO_SHA1); + safexcel_crypto_register(sc, CRYPTO_SHA1_HMAC); + safexcel_crypto_register(sc, CRYPTO_SHA2_224); + safexcel_crypto_register(sc, CRYPTO_SHA2_224_HMAC); + safexcel_crypto_register(sc, CRYPTO_SHA2_256); + safexcel_crypto_register(sc, CRYPTO_SHA2_256_HMAC); + safexcel_crypto_register(sc, CRYPTO_SHA2_384); + safexcel_crypto_register(sc, CRYPTO_SHA2_384_HMAC); + safexcel_crypto_register(sc, CRYPTO_SHA2_512); + safexcel_crypto_register(sc, CRYPTO_SHA2_512_HMAC); + return (0); err2: @@ -1256,7 +1284,6 @@ safexcel_detach(device_t dev) static int safexcel_set_context(struct safexcel_request *req) { - const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_context_record *ctx; struct safexcel_session *sess; @@ -1264,58 +1291,51 @@ safexcel_set_context(struct safexcel_request *req) int off; crp = req->crp; - csp = crypto_get_params(crp->crp_session); sess = req->sess; ctx = (struct safexcel_context_record *)req->ctx.vaddr; data = (uint8_t *)ctx->data; - if (csp->csp_cipher_alg != 0) { - if (crp->crp_cipher_key != NULL) - memcpy(data, crp->crp_cipher_key, sess->klen); + if (req->enc != NULL) { + if ((req->enc->crd_flags & CRD_F_KEY_EXPLICIT) != 0) + memcpy(data, req->enc->crd_key, sess->klen); else - memcpy(data, csp->csp_cipher_key, sess->klen); + memcpy(data, sess->key, sess->klen); off = sess->klen; - } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { - if (crp->crp_auth_key != NULL) - memcpy(data, crp->crp_auth_key, sess->klen); - else - memcpy(data, csp->csp_auth_key, sess->klen); - off = sess->klen; } else { off = 0; } - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_NIST_GCM_16: - memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); - off += GMAC_BLOCK_LEN; - break; - case CRYPTO_AES_CCM_16: - memcpy(data + off, sess->xcbc_key, - AES_BLOCK_LEN * 2 + sess->klen); - off += AES_BLOCK_LEN * 2 + sess->klen; - break; - case CRYPTO_AES_XTS: - memcpy(data + off, sess->tweak_key, sess->klen); - off += sess->klen; - break; + if (req->enc != NULL) { + switch (req->enc->crd_alg) { + case CRYPTO_AES_NIST_GCM_16: + memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); + off += GMAC_BLOCK_LEN; + break; + case CRYPTO_AES_CCM_16: + memcpy(data + off, sess->xcbc_key, + AES_BLOCK_LEN * 2 + sess->klen); + off += AES_BLOCK_LEN * 2 + sess->klen; + break; + case CRYPTO_AES_XTS: + memcpy(data + off, sess->tweak_key, sess->klen); + off += sess->klen; + break; + } } - switch (csp->csp_auth_alg) { - case CRYPTO_AES_NIST_GMAC: - memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); - off += GMAC_BLOCK_LEN; - break; - case CRYPTO_SHA1_HMAC: - case CRYPTO_SHA2_224_HMAC: - case CRYPTO_SHA2_256_HMAC: - case CRYPTO_SHA2_384_HMAC: - case CRYPTO_SHA2_512_HMAC: - memcpy(data + off, sess->hmac_ipad, sess->statelen); - off += sess->statelen; - memcpy(data + off, sess->hmac_opad, sess->statelen); - off += sess->statelen; - break; + if (req->mac != NULL) { + switch (req->mac->crd_alg) { + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_224_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + memcpy(data + off, sess->hmac_ipad, sess->statelen); + off += sess->statelen; + memcpy(data + off, sess->hmac_opad, sess->statelen); + off += sess->statelen; + break; + } } return (off); @@ -1331,14 +1351,12 @@ static void safexcel_set_command(struct safexcel_request *req, struct safexcel_cmd_descr *cdesc) { - const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_session *sess; uint32_t ctrl0, ctrl1, ctxr_len; int alg; crp = req->crp; - csp = crypto_get_params(crp->crp_session); sess = req->sess; ctrl0 = sess->alg | sess->digest | sess->hash; @@ -1347,13 +1365,14 @@ safexcel_set_command(struct safexcel_request *req, ctxr_len = safexcel_set_context(req) / sizeof(uint32_t); ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len); - alg = csp->csp_cipher_alg; - if (alg == 0) - alg = csp->csp_auth_alg; + if (req->enc != NULL) + alg = req->enc->crd_alg; + else + alg = req->mac->crd_alg; switch (alg) { case CRYPTO_AES_CCM_16: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT | SAFEXCEL_CONTROL0_KEY_EN; } else { @@ -1366,23 +1385,23 @@ safexcel_set_command(struct safexcel_request *req, case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN; - if (csp->csp_auth_alg != 0) + if (req->mac != NULL) ctrl0 |= SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT; } else { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN | SAFEXCEL_CONTROL0_KEY_EN; - if (csp->csp_auth_alg != 0) + if (req->mac != NULL) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; + ctrl1 |= SAFEXCEL_CONTROL1_HASH_STORE; + } } break; case CRYPTO_AES_NIST_GCM_16: - case CRYPTO_AES_NIST_GMAC: - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) || - csp->csp_auth_alg != 0) { + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) { ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT | SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_OUT; @@ -1391,7 +1410,8 @@ safexcel_set_command(struct safexcel_request *req, SAFEXCEL_CONTROL0_KEY_EN | SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN; } - if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { + if (req->enc != NULL && + req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16) { ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE | SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 | SAFEXCEL_CONTROL1_IV2; @@ -1454,11 +1474,28 @@ safexcel_instr_insert_digest(struct safexcel_instr **i *instrp = instr + 1; } +static void +safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, struct safexcel_request *req, int len) +{ + struct safexcel_instr *instr; + + instr = *instrp; + + instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; + instr->length = len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | + SAFEXCEL_INSTR_STATUS_LAST_PACKET; + instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST | + SAFEXCEL_INSTR_DEST_OUTPUT; + + *instrp = instr + 1; +} + /* * Retrieve and verify a digest. */ static void -safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len) +safexcel_instr_verify_digest(struct safexcel_instr **instrp, int len) { struct safexcel_instr *instr; @@ -1507,13 +1544,9 @@ static void safexcel_instr_cipher(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { - struct cryptop *crp; - - crp = req->crp; - /* Insert the payload. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; + instr->length = req->enc->crd_len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | @@ -1526,42 +1559,27 @@ static void safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc) { - const struct crypto_session_params *csp; - struct cryptop *crp; struct safexcel_instr *start; - crp = req->crp; - csp = crypto_get_params(crp->crp_session); start = instr; - /* Insert the AAD. */ + /* Encrypt any data left in the request. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_aad_length; - instr->status = crp->crp_payload_length == 0 ? - SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; + instr->length = req->enc->crd_len; + instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_INS_LAST | - SAFEXCEL_INSTR_DEST_HASH; + SAFEXCEL_INSTR_DEST_CRYPTO | + SAFEXCEL_INSTR_DEST_HASH | + SAFEXCEL_INSTR_DEST_OUTPUT; instr++; - /* Encrypt any data left in the request. */ - if (crp->crp_payload_length > 0) { - instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; - instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; - instr->instructions = SAFEXCEL_INSTR_INS_LAST | - SAFEXCEL_INSTR_DEST_CRYPTO | - SAFEXCEL_INSTR_DEST_HASH | - SAFEXCEL_INSTR_DEST_OUTPUT; - instr++; - } - /* * Compute the digest, or extract it and place it in the output stream. */ - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else - safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + safexcel_instr_retrieve_digest(&instr, req, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } @@ -1577,7 +1595,7 @@ safexcel_instr_sha_hash(struct safexcel_request *req, /* Pass the input data to the hash engine. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; + instr->length = req->mac->crd_len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; @@ -1624,7 +1642,7 @@ safexcel_instr_ccm(struct safexcel_request *req, struc * Insert B0 and the AAD length into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; - instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); + instr->length = blen + (req->mac->crd_len > 0 ? 2 : 0); instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH | SAFEXCEL_INSTR_INSERT_IMMEDIATE; @@ -1635,33 +1653,33 @@ safexcel_instr_ccm(struct safexcel_request *req, struc b0[0] = L - 1 | /* payload length size */ ((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ | - (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; + (req->mac->crd_len > 0 ? 1 : 0) << 6 /* AAD present bit */; memcpy(&b0[1], req->iv, AES_CCM_IV_LEN); - b0[14] = crp->crp_payload_length >> 8; - b0[15] = crp->crp_payload_length & 0xff; + b0[14] = req->enc->crd_len >> 8; + b0[15] = req->enc->crd_len & 0xff; instr += blen / sizeof(*instr); /* Insert the AAD length and data into the input stream. */ - if (crp->crp_aad_length > 0) { + if (req->mac->crd_len > 0) { alenp = (uint8_t *)instr; - alenp[0] = crp->crp_aad_length >> 8; - alenp[1] = crp->crp_aad_length & 0xff; + alenp[0] = req->mac->crd_len >> 8; + alenp[1] = req->mac->crd_len & 0xff; alenp[2] = 0; alenp[3] = 0; instr++; instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_aad_length; + instr->length = req->mac->crd_len; instr->status = 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; /* Insert zero padding. */ - aalign = (crp->crp_aad_length + 2) & (blen - 1); + aalign = (req->mac->crd_len + 2) & (blen - 1); instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = aalign == 0 ? 0 : - blen - ((crp->crp_aad_length + 2) & (blen - 1)); - instr->status = crp->crp_payload_length == 0 ? + blen - ((req->mac->crd_len + 2) & (blen - 1)); + instr->status = req->enc->crd_len == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; @@ -1670,10 +1688,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ - if (crp->crp_payload_length > 0) { + if (req->enc->crd_len > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; - instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? + instr->length = req->enc->crd_len; + instr->status = (req->enc->crd_len & (blen - 1)) == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | @@ -1682,10 +1700,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc instr++; /* Insert zero padding. */ - if (crp->crp_payload_length & (blen - 1)) { + if (req->enc->crd_len & (blen - 1)) { instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; instr->length = blen - - (crp->crp_payload_length & (blen - 1)); + (req->enc->crd_len & (blen - 1)); instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_HASH; instr++; @@ -1695,10 +1713,10 @@ safexcel_instr_ccm(struct safexcel_request *req, struc /* * Compute the digest, or extract it and place it in the output stream. */ - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else - safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + safexcel_instr_verify_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } @@ -1718,8 +1736,8 @@ safexcel_instr_gcm(struct safexcel_request *req, struc /* Insert the AAD into the input stream. */ instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_aad_length; - instr->status = crp->crp_payload_length == 0 ? + instr->length = req->mac->crd_len; + instr->status = req->enc->crd_len == 0 ? SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; instr->instructions = SAFEXCEL_INSTR_INS_LAST | SAFEXCEL_INSTR_DEST_HASH; @@ -1728,9 +1746,9 @@ safexcel_instr_gcm(struct safexcel_request *req, struc safexcel_instr_temp_aes_block(&instr); /* Insert the cipher payload into the input stream. */ - if (crp->crp_payload_length > 0) { + if (req->enc->crd_len > 0) { instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; + instr->length = req->enc->crd_len; instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | @@ -1741,51 +1759,22 @@ safexcel_instr_gcm(struct safexcel_request *req, struc /* * Compute the digest, or extract it and place it in the output stream. */ - if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + if ((req->enc->crd_flags & CRD_F_ENCRYPT) != 0) safexcel_instr_insert_digest(&instr, req->sess->digestlen); else - safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); + safexcel_instr_verify_digest(&instr, req->sess->digestlen); cdesc->additional_cdata_size = instr - start; } static void -safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, - struct safexcel_cmd_descr *cdesc) -{ - struct cryptop *crp; - struct safexcel_instr *start; - - memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); - cdesc->control_data.token[3] = htobe32(1); - - crp = req->crp; - start = instr; - - instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; - instr->length = crp->crp_payload_length; - instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; - instr->instructions = SAFEXCEL_INSTR_INS_LAST | - SAFEXCEL_INSTR_DEST_HASH; - instr++; - - safexcel_instr_temp_aes_block(&instr); - - safexcel_instr_insert_digest(&instr, req->sess->digestlen); - - cdesc->additional_cdata_size = instr - start; -} - -static void safexcel_set_token(struct safexcel_request *req) { - const struct crypto_session_params *csp; struct safexcel_cmd_descr *cdesc; struct safexcel_instr *instr; struct safexcel_softc *sc; int ringidx; - csp = crypto_get_params(req->crp->crp_session); cdesc = req->cdesc; sc = req->sc; ringidx = req->sess->ringidx; @@ -1797,8 +1786,7 @@ safexcel_set_token(struct safexcel_request *req) * in the token itself. Otherwise we use an additional token descriptor * and the embedded instruction space is used to store the IV. */ - if (csp->csp_cipher_alg == 0 && - csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { + if (req->enc == NULL) { instr = (void *)cdesc->control_data.token; } else { instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + @@ -1807,30 +1795,32 @@ safexcel_set_token(struct safexcel_request *req) cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; } - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_NIST_GCM_16: - safexcel_instr_gcm(req, instr, cdesc); - break; - case CRYPTO_AES_CCM_16: - safexcel_instr_ccm(req, instr, cdesc); - break; - case CRYPTO_AES_XTS: - memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); - memset(cdesc->control_data.token + - AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); + if (req->enc != NULL) { + switch (req->enc->crd_alg) { + case CRYPTO_AES_NIST_GCM_16: + safexcel_instr_gcm(req, instr, cdesc); + break; + case CRYPTO_AES_CCM_16: + safexcel_instr_ccm(req, instr, cdesc); + break; + case CRYPTO_AES_XTS: + memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); + memset(cdesc->control_data.token + + AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); - safexcel_instr_cipher(req, instr, cdesc); - break; - case CRYPTO_AES_CBC: - case CRYPTO_AES_ICM: - memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); - if (csp->csp_auth_alg != 0) - safexcel_instr_eta(req, instr, cdesc); - else safexcel_instr_cipher(req, instr, cdesc); - break; - default: - switch (csp->csp_auth_alg) { + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); + if (req->mac != NULL) + safexcel_instr_eta(req, instr, cdesc); + else + safexcel_instr_cipher(req, instr, cdesc); + break; + } + } else { + switch (req->mac->crd_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: @@ -1843,13 +1833,9 @@ safexcel_set_token(struct safexcel_request *req) case CRYPTO_SHA2_512_HMAC: safexcel_instr_sha_hash(req, instr); break; - case CRYPTO_AES_NIST_GMAC: - safexcel_instr_gmac(req, instr, cdesc); - break; default: - panic("unhandled auth request %d", csp->csp_auth_alg); + panic("unhandled auth request %d", req->mac->crd_alg); } - break; } } @@ -1896,8 +1882,8 @@ safexcel_cmd_descr_add(struct safexcel_ring *ring, boo struct safexcel_cmd_descr *cdesc; struct safexcel_cmd_descr_ring *cring; - KASSERT(full_data_len <= SAFEXCEL_MAX_REQUEST_SIZE, - ("%s: request length %u too long", __func__, full_data_len)); + KASSERT(reqlen <= SAFEXCEL_MAX_REQUEST_SIZE, + ("%s: request length %u too long", __func__, reqlen)); mtx_assert(&ring->mtx, MA_OWNED); cring = &ring->cdr; @@ -1978,13 +1964,14 @@ safexcel_append_segs(bus_dma_segment_t *segs, int nseg len -= seglen; start = 0; } + + KASSERT(len == 0, ("%s: %d residual bytes", __func__, len)); } static void safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { - const struct crypto_session_params *csp; struct cryptop *crp; struct safexcel_cmd_descr *cdesc; struct safexcel_request *req; @@ -2002,7 +1989,6 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t } crp = req->crp; - csp = crypto_get_params(crp->crp_session); sess = req->sess; ring = &req->sc->sc_ring[sess->ringidx]; @@ -2021,23 +2007,30 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t */ sglist_reset(ring->cmd_data); sglist_reset(ring->res_data); - if (crp->crp_aad_length != 0) { + if (req->mac != NULL && (req->enc == NULL || + req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16 || + req->enc->crd_alg == CRYPTO_AES_CCM_16)) { safexcel_append_segs(segs, nseg, ring->cmd_data, - crp->crp_aad_start, crp->crp_aad_length); + req->mac->crd_skip, req->mac->crd_len); } - safexcel_append_segs(segs, nseg, ring->cmd_data, - crp->crp_payload_start, crp->crp_payload_length); - if (csp->csp_cipher_alg != 0) { + if (req->enc != NULL) { + safexcel_append_segs(segs, nseg, ring->cmd_data, + req->enc->crd_skip, req->enc->crd_len); safexcel_append_segs(segs, nseg, ring->res_data, - crp->crp_payload_start, crp->crp_payload_length); + req->enc->crd_skip, req->enc->crd_len); } if (sess->digestlen > 0) { - if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { + if (req->enc == NULL || + (req->enc->crd_flags & CRD_F_ENCRYPT) != 0) + safexcel_append_segs(segs, nseg, ring->res_data, + req->mac->crd_inject, sess->digestlen); + else if (req->enc->crd_alg == CRYPTO_AES_NIST_GCM_16 || + req->enc->crd_alg == CRYPTO_AES_CCM_16) { safexcel_append_segs(segs, nseg, ring->cmd_data, - crp->crp_digest_start, sess->digestlen); + req->mac->crd_inject, sess->digestlen); } else { safexcel_append_segs(segs, nseg, ring->res_data, - crp->crp_digest_start, sess->digestlen); + req->mac->crd_inject, sess->digestlen); } } @@ -2097,63 +2090,84 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t req->rdescs = sg->sg_nseg; } +static void +safexcel_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, + bus_size_t mapsize __unused, int error) +{ + safexcel_create_chain_cb(arg, segs, nseg, error); +} + +#include <sys/uio.h> static int safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req) { + struct cryptop *crp; int error; req->error = 0; req->cdescs = req->rdescs = 0; + crp = req->crp; - error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, - safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); + if ((crp->crp_flags & CRYPTO_F_IOV) != 0) { + error = bus_dmamap_load_uio(ring->data_dtag, req->dmap, + (struct uio *)crp->crp_buf, safexcel_create_chain_cb2, + req, BUS_DMA_NOWAIT); + } else if ((crp->crp_flags & CRYPTO_F_IMBUF) != 0) { + error = bus_dmamap_load_mbuf(ring->data_dtag, req->dmap, + (struct mbuf *)crp->crp_buf, safexcel_create_chain_cb2, + req, BUS_DMA_NOWAIT); + } else { + error = bus_dmamap_load(ring->data_dtag, req->dmap, + crp->crp_buf, crp->crp_ilen, + safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); + } if (error == 0) req->dmap_loaded = true; - - if (req->error != 0) + else if (req->error != 0) error = req->error; - return (error); } -static bool -safexcel_probe_cipher(const struct crypto_session_params *csp) -{ - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_CBC: - case CRYPTO_AES_ICM: - if (csp->csp_ivlen != AES_BLOCK_LEN) - return (false); - break; - case CRYPTO_AES_XTS: - if (csp->csp_ivlen != AES_XTS_IV_LEN) - return (false); - break; - default: - return (false); - } - - return (true); -} - /* * Determine whether the driver can implement a session with the requested * parameters. */ static int -safexcel_probesession(device_t dev, const struct crypto_session_params *csp) +safexcel_probesession(struct cryptoini *enc, struct cryptoini *mac) { - switch (csp->csp_mode) { - case CSP_MODE_CIPHER: - if (!safexcel_probe_cipher(csp)) - return (EINVAL); - break; - case CSP_MODE_DIGEST: - switch (csp->csp_auth_alg) { - case CRYPTO_AES_NIST_GMAC: - if (csp->csp_ivlen != AES_GCM_IV_LEN) + if (enc != NULL) { + switch (enc->cri_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (mac == NULL || + (mac->cri_alg != CRYPTO_AES_128_NIST_GMAC && + mac->cri_alg != CRYPTO_AES_192_NIST_GMAC && + mac->cri_alg != CRYPTO_AES_256_NIST_GMAC)) return (EINVAL); break; + case CRYPTO_AES_CCM_16: + if (mac == NULL || + mac->cri_alg != CRYPTO_AES_CCM_CBC_MAC) + return (EINVAL); + break; + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + if (mac != NULL && + mac->cri_alg != CRYPTO_SHA1_HMAC && + mac->cri_alg != CRYPTO_SHA2_224_HMAC && + mac->cri_alg != CRYPTO_SHA2_256_HMAC && + mac->cri_alg != CRYPTO_SHA2_384_HMAC && + mac->cri_alg != CRYPTO_SHA2_512_HMAC) + return (EINVAL); + break; + case CRYPTO_AES_XTS: + if (mac != NULL) + return (EINVAL); + break; + default: + return (EINVAL); + } + } else { + switch (mac->cri_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_224: @@ -2168,47 +2182,9 @@ safexcel_probesession(device_t dev, const struct crypt default: return (EINVAL); } - break; - case CSP_MODE_AEAD: - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_NIST_GCM_16: - if (csp->csp_ivlen != AES_GCM_IV_LEN) - return (EINVAL); - break; - case CRYPTO_AES_CCM_16: - if (csp->csp_ivlen != AES_CCM_IV_LEN) - return (EINVAL); - break; - default: - return (EINVAL); - } - break; - case CSP_MODE_ETA: - if (!safexcel_probe_cipher(csp)) - return (EINVAL); - switch (csp->csp_cipher_alg) { - case CRYPTO_AES_CBC: - case CRYPTO_AES_ICM: - /* - * The EIP-97 does not support combining AES-XTS with - * hash operations. - */ - if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && - csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && - csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && - csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && - csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) - return (EINVAL); - break; - default: - return (EINVAL); - } - break; - default: - return (EINVAL); } - return (CRYPTODEV_PROBE_HARDWARE); + return (0); } /* @@ -2286,20 +2262,64 @@ safexcel_setkey_hmac_digest(struct auth_hash *ahash, u } } +static void +safexcel_hmac_init_pad(struct auth_hash *axf, const char *key, int klen, + union authctx *auth_ctx, uint8_t padval) +{ + uint8_t hmac_key[HMAC_MAX_BLOCK_LEN]; + u_int i; + + memset(hmac_key, 0, sizeof(hmac_key)); + if (klen > axf->blocksize) { + axf->Init(auth_ctx); + axf->Update(auth_ctx, key, klen); + axf->Final(hmac_key, auth_ctx); + klen = axf->hashsize; + } else { + memcpy(hmac_key, key, klen); + } + + for (i = 0; i < axf->blocksize; i++) + hmac_key[i] ^= padval; + + axf->Init(auth_ctx); + axf->Update(auth_ctx, hmac_key, axf->blocksize); + explicit_bzero(hmac_key, sizeof(hmac_key)); +} + /* * Pre-compute the inner and outer digests used in the HMAC algorithm. */ static void -safexcel_setkey_hmac(const struct crypto_session_params *csp, - struct safexcel_session *sess, const uint8_t *key, int klen) +safexcel_setkey_hmac(struct safexcel_session *sess, int alg, const uint8_t *key, + int klen) { union authctx ctx; struct auth_hash *ahash; - ahash = crypto_auth_hash(csp); - hmac_init_ipad(ahash, key, klen, &ctx); + switch (alg) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202007221422.06MEMZUK013536>