From nobody Fri Mar 18 15:41:53 2022 X-Original-To: dev-commits-src-branches@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 1C7DB1A31409; Fri, 18 Mar 2022 15:41:54 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4KKpCj6z9Mz3vdT; Fri, 18 Mar 2022 15:41:53 +0000 (UTC) (envelope-from git@FreeBSD.org) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1647618114; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=wAjAP/eX0rd8tRT12n6PeX3a39T+Lgo+sExWOErcGsM=; b=AOSYEJcWElKQbErUPHfpYXxfNLXJ0nTrqQWnkjaqHCc+uIgeqchvanC6t8xwZUk1lelFcV rY/VOjJ8sxHJP3Bf+u3ss1Voym68Mx25Z2lT9+gkMyHEqhWrLx1hCs4+pTPhQLqcdP6pF9 Am++4ZdBCBRY/VVWV9ZpSZD8mh1W1qbERGw6oAiOh26GecBL6uCXni94LxbFgSf2YCti8z 6SPojSYJ74NJD9b9QrVHvrT3Vb9zSQD83syJwSBssHmgOy4KrUA7UGb26hW2rBbmHLkQoQ q5TfmjvvrqdLKtU6NIEINto66o+izsFxWeTHffeI9YsTuAAx4wGAkJR4hR13cg== Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id CF938219A5; Fri, 18 Mar 2022 15:41:53 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 22IFfrMd089191; Fri, 18 Mar 2022 15:41:53 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 22IFfrub089190; Fri, 18 Mar 2022 15:41:53 GMT (envelope-from git) Date: Fri, 18 Mar 2022 15:41:53 GMT Message-Id: <202203181541.22IFfrub089190@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mark Johnston Subject: git: 04df02b2f42e - stable/13 - armv8crypto: Factor out some duplicated GCM code List-Id: Commits to the stable branches of the FreeBSD src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-branches List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-branches@freebsd.org X-BeenThere: dev-commits-src-branches@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: markj X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 04df02b2f42ecb78bbc38b94c4764ee5eda385c2 Auto-Submitted: auto-generated ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org; s=dkim; t=1647618114; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=wAjAP/eX0rd8tRT12n6PeX3a39T+Lgo+sExWOErcGsM=; b=rb0EmOEOrhOcnDFPvlXo6t70MqTGLT1wtDUyL7VS+WpxCSbCetBuLDT9M3aGzFFKlyzwHp eNV1yPcOzPDH5Po4fg7+wvCUB1f6ljmCFb2PHjyBHKDGWm2w1DJnAPcDjvY43wc4QZnXaW UdsYfScVW+HMkANlKqIB4eOGyXWiIb9onArZ96ZSAFUpJQPhVFnRoJ6UeXWSOcwQmTx5m+ l1oimTlCDjBhG50SWxZIE2yuX0EU2AtxZ8FxRbyq1pXTxOTA53ErHL/ssk3wrVQtIVKdQQ BN0DHaZjdC/U9wzzEdgHJqVGjKxETSFst34p0suc6RTu7FPdMAj3B5ulDPQ9ZA== ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1647618114; a=rsa-sha256; cv=none; b=lZ5hvCgMZC9SpdlLzds0GjBcvvwFcXe1fGAUNV4y+ruenT3WVXotnd+STe+wQiUBqSyZJT AORLDCJ6AnDbxC4FG26PyhGntcvGJKizsKQAhGDxsiD+KRgXgbXpMYbhlTso968elyAfLU P4n0TuJQhBgKyDOA3YQBw1e1EQhyuwZcB7oIAl7Ih2Rc3g8qYPQmJbCHYWq5oig65BW6PP 40dnMb5HapxKGZAsA8fhN6JuM8yHCa/IEF41EIfeIseWFVkYBV70ypAS/DeJbKgWgVqkyq G4GEcrcPdVvFMtioVGtiri3Sv3npeKRNOidyWJcUxMbU+2lRIBUWmM+/mpjSJA== ARC-Authentication-Results: i=1; mx1.freebsd.org; none X-ThisMailContainsUnwantedMimeParts: N The branch stable/13 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=04df02b2f42ecb78bbc38b94c4764ee5eda385c2 commit 04df02b2f42ecb78bbc38b94c4764ee5eda385c2 Author: Mark Johnston AuthorDate: 2022-02-16 02:45:59 +0000 Commit: Mark Johnston CommitDate: 2022-03-18 15:29:24 +0000 armv8crypto: Factor out some duplicated GCM code This is in preparation for using buffer cursors. No functional change intended. Reviewed by: jhb Sponsored by: Ampere Computing LLC Submitted by: Klara Inc. (cherry picked from commit 0b3235ef743d1561c57989042b3c364a5a955f4f) --- sys/crypto/armv8/armv8_crypto_wrap.c | 111 ++++++++++++++++------------------- 1 file changed, 51 insertions(+), 60 deletions(-) diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c index 3c0223964ee4..b5aee0cc1cf6 100644 --- a/sys/crypto/armv8/armv8_crypto_wrap.c +++ b/sys/crypto/armv8/armv8_crypto_wrap.c @@ -249,46 +249,71 @@ struct armv8_gcm_state { uint8_t aes_counter[AES_BLOCK_LEN]; }; -void -armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, - const uint8_t *from, uint8_t *to, - size_t authdatalen, const uint8_t *authdata, - uint8_t tag[static GMAC_DIGEST_LEN], - const uint8_t iv[static AES_GCM_IV_LEN], - const __uint128_val_t *Htable) +static void +armv8_aes_gmac_setup(struct armv8_gcm_state *s, AES_key_t *aes_key, + const uint8_t *authdata, size_t authdatalen, + const uint8_t iv[static AES_GCM_IV_LEN], const __uint128_val_t *Htable) { - struct armv8_gcm_state s; - const uint64_t *from64; - uint64_t *to64; uint8_t block[AES_BLOCK_LEN]; - size_t i, trailer; + size_t trailer; - bzero(&s.aes_counter, AES_BLOCK_LEN); - memcpy(s.aes_counter, iv, AES_GCM_IV_LEN); + bzero(s->aes_counter, AES_BLOCK_LEN); + memcpy(s->aes_counter, iv, AES_GCM_IV_LEN); /* Setup the counter */ - s.aes_counter[AES_BLOCK_LEN - 1] = 1; + s->aes_counter[AES_BLOCK_LEN - 1] = 1; /* EK0 for a final GMAC round */ - aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key); + aes_v8_encrypt(s->aes_counter, s->EK0.c, aes_key); /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ - s.aes_counter[AES_BLOCK_LEN - 1] = 2; + s->aes_counter[AES_BLOCK_LEN - 1] = 2; - memset(s.Xi.c, 0, sizeof(s.Xi.c)); + memset(s->Xi.c, 0, sizeof(s->Xi.c)); trailer = authdatalen % AES_BLOCK_LEN; if (authdatalen - trailer > 0) { - gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer); + gcm_ghash_v8(s->Xi.u, Htable, authdata, authdatalen - trailer); authdata += authdatalen - trailer; } if (trailer > 0 || authdatalen == 0) { memset(block, 0, sizeof(block)); memcpy(block, authdata, trailer); - gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); + gcm_ghash_v8(s->Xi.u, Htable, block, AES_BLOCK_LEN); } +} - from64 = (const uint64_t*)from; - to64 = (uint64_t*)to; +static void +armv8_aes_gmac_finish(struct armv8_gcm_state *s, size_t len, + size_t authdatalen, const __uint128_val_t *Htable) +{ + /* Lengths block */ + s->lenblock.u[0] = s->lenblock.u[1] = 0; + s->lenblock.d[1] = htobe32(authdatalen * 8); + s->lenblock.d[3] = htobe32(len * 8); + gcm_ghash_v8(s->Xi.u, Htable, s->lenblock.c, AES_BLOCK_LEN); + + s->Xi.u[0] ^= s->EK0.u[0]; + s->Xi.u[1] ^= s->EK0.u[1]; +} + +void +armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, + const uint8_t *from, uint8_t *to, + size_t authdatalen, const uint8_t *authdata, + uint8_t tag[static GMAC_DIGEST_LEN], + const uint8_t iv[static AES_GCM_IV_LEN], + const __uint128_val_t *Htable) +{ + struct armv8_gcm_state s; + const uint64_t *from64; + uint64_t *to64; + uint8_t block[AES_BLOCK_LEN]; + size_t i, trailer; + + armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable); + + from64 = (const uint64_t *)from; + to64 = (uint64_t *)to; trailer = len % AES_BLOCK_LEN; for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { @@ -316,14 +341,7 @@ armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } - /* Lengths block */ - s.lenblock.u[0] = s.lenblock.u[1] = 0; - s.lenblock.d[1] = htobe32(authdatalen * 8); - s.lenblock.d[3] = htobe32(len * 8); - gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN); - - s.Xi.u[0] ^= s.EK0.u[0]; - s.Xi.u[1] ^= s.EK0.u[1]; + armv8_aes_gmac_finish(&s, len, authdatalen, Htable); memcpy(tag, s.Xi.c, GMAC_DIGEST_LEN); explicit_bzero(&s, sizeof(s)); @@ -345,26 +363,8 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, int error; error = 0; - bzero(&s.aes_counter, AES_BLOCK_LEN); - memcpy(s.aes_counter, iv, AES_GCM_IV_LEN); - - /* Setup the counter */ - s.aes_counter[AES_BLOCK_LEN - 1] = 1; - - /* EK0 for a final GMAC round */ - aes_v8_encrypt(s.aes_counter, s.EK0.c, aes_key); - memset(s.Xi.c, 0, sizeof(s.Xi.c)); - trailer = authdatalen % AES_BLOCK_LEN; - if (authdatalen - trailer > 0) { - gcm_ghash_v8(s.Xi.u, Htable, authdata, authdatalen - trailer); - authdata += authdatalen - trailer; - } - if (trailer > 0 || authdatalen == 0) { - memset(block, 0, sizeof(block)); - memcpy(block, authdata, trailer); - gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); - } + armv8_aes_gmac_setup(&s, aes_key, authdata, authdatalen, iv, Htable); trailer = len % AES_BLOCK_LEN; if (len - trailer > 0) @@ -375,24 +375,15 @@ armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, gcm_ghash_v8(s.Xi.u, Htable, block, AES_BLOCK_LEN); } - /* Lengths block */ - s.lenblock.u[0] = s.lenblock.u[1] = 0; - s.lenblock.d[1] = htobe32(authdatalen * 8); - s.lenblock.d[3] = htobe32(len * 8); - gcm_ghash_v8(s.Xi.u, Htable, s.lenblock.c, AES_BLOCK_LEN); + armv8_aes_gmac_finish(&s, len, authdatalen, Htable); - s.Xi.u[0] ^= s.EK0.u[0]; - s.Xi.u[1] ^= s.EK0.u[1]; if (timingsafe_bcmp(tag, s.Xi.c, GMAC_DIGEST_LEN) != 0) { error = EBADMSG; goto out; } - /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ - s.aes_counter[AES_BLOCK_LEN - 1] = 2; - - from64 = (const uint64_t*)from; - to64 = (uint64_t*)to; + from64 = (const uint64_t *)from; + to64 = (uint64_t *)to; for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { aes_v8_encrypt(s.aes_counter, s.EKi.c, aes_key);