Date: Thu, 7 Jan 2021 19:36:21 GMT From: Mitchell Horne <mhorne@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 4979620ece98 - main - armv8crypto: add AES-XTS support Message-ID: <202101071936.107JaL8B069216@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by mhorne: URL: https://cgit.FreeBSD.org/src/commit/?id=4979620ece984ffb10c27c2db7d0e253eb84b2ba commit 4979620ece984ffb10c27c2db7d0e253eb84b2ba Author: Mitchell Horne <mhorne@FreeBSD.org> AuthorDate: 2021-01-07 19:30:40 +0000 Commit: Mitchell Horne <mhorne@FreeBSD.org> CommitDate: 2021-01-07 19:35:20 +0000 armv8crypto: add AES-XTS support A straightforward(ish) port from aesni(4). This implementation does not perform loop unrolling on the input blocks, so this is left as a future performance improvement. Submitted by: Greg V <greg AT unrelenting.technology> Looks good: jhb, jmg Tested by: mhorne Differential Revision: https://reviews.freebsd.org/D21017 --- sys/crypto/armv8/armv8_crypto.c | 51 +++++++++++++++----- sys/crypto/armv8/armv8_crypto.h | 6 +++ sys/crypto/armv8/armv8_crypto_wrap.c | 92 ++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 11 deletions(-) diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c index 43dafd7b365c..90d096b53b1a 100644 --- a/sys/crypto/armv8/armv8_crypto.c +++ b/sys/crypto/armv8/armv8_crypto.c @@ -114,7 +114,7 @@ armv8_crypto_probe(device_t dev) break; } - device_set_desc_copy(dev, "AES-CBC"); + device_set_desc_copy(dev, "AES-CBC,AES-XTS"); /* TODO: Check more fields as we support more features */ @@ -204,6 +204,17 @@ armv8_crypto_probesession(device_t dev, return (EINVAL); } break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return (EINVAL); + switch (csp->csp_cipher_klen * 8) { + case 256: + case 512: + break; + default: + return (EINVAL); + } + break; default: return (EINVAL); } @@ -211,16 +222,19 @@ armv8_crypto_probesession(device_t dev, default: return (EINVAL); } - return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); + return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } static void armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, - const struct crypto_session_params *csp) + const struct crypto_session_params *csp, const uint8_t *key, int keylen) { int i; - switch (csp->csp_cipher_klen * 8) { + if (csp->csp_cipher_alg == CRYPTO_AES_XTS) + keylen /= 2; + + switch (keylen * 8) { case 128: ses->rounds = AES128_ROUNDS; break; @@ -231,16 +245,19 @@ armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, ses->rounds = AES256_ROUNDS; break; default: - panic("invalid CBC key length"); + panic("invalid AES key length"); } - rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key, - csp->csp_cipher_klen * 8); - rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key, - csp->csp_cipher_klen * 8); + rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8); + rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8); + if (csp->csp_cipher_alg == CRYPTO_AES_XTS) + rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8); + for (i = 0; i < nitems(ses->enc_schedule); i++) { ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]); ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]); + if (csp->csp_cipher_alg == CRYPTO_AES_XTS) + ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]); } } @@ -259,7 +276,8 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses, } ses = crypto_get_driver_session(cses); - armv8_crypto_cipher_setup(ses, csp); + armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, + csp->csp_cipher_klen); rw_wunlock(&sc->lock); return (0); } @@ -333,7 +351,8 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses, } if (crp->crp_cipher_key != NULL) { - panic("armv8: new cipher key"); + armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, + csp->csp_cipher_klen); } crypto_read_iv(crp, iv); @@ -348,6 +367,16 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses, armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule, crp->crp_payload_length, buf, iv); break; + case CRYPTO_AES_XTS: + if (encflag) + armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule, + ses->xts_schedule, crp->crp_payload_length, buf, + buf, iv); + else + armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule, + ses->xts_schedule, crp->crp_payload_length, buf, + buf, iv); + break; } if (allocated) diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h index 01d31f7cec78..2d0be163b072 100644 --- a/sys/crypto/armv8/armv8_crypto.h +++ b/sys/crypto/armv8/armv8_crypto.h @@ -40,6 +40,7 @@ struct armv8_crypto_session { uint32_t enc_schedule[AES_SCHED_LEN/4]; uint32_t dec_schedule[AES_SCHED_LEN/4]; + uint32_t xts_schedule[AES_SCHED_LEN/4]; int algo; int rounds; }; @@ -49,4 +50,9 @@ void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *, void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *, const uint8_t[static AES_BLOCK_LEN]); +void armv8_aes_encrypt_xts(int, const void *, const void *, size_t, + const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); +void armv8_aes_decrypt_xts(int, const void *, const void *, size_t, + const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); + #endif /* _ARMV8_CRYPTO_H_ */ diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c index 643b2be2cb12..83af3fad40ef 100644 --- a/sys/crypto/armv8/armv8_crypto_wrap.c +++ b/sys/crypto/armv8/armv8_crypto_wrap.c @@ -126,3 +126,95 @@ armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len, buf += AES_BLOCK_LEN; } } + +#define AES_XTS_BLOCKSIZE 16 +#define AES_XTS_IVSIZE 8 +#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ + +static inline int32x4_t +xts_crank_lfsr(int32x4_t inp) +{ + const int32x4_t alphamask = {AES_XTS_ALPHA, 1, 1, 1}; + int32x4_t xtweak, ret; + + /* set up xor mask */ + xtweak = vextq_s32(inp, inp, 3); + xtweak = vshrq_n_s32(xtweak, 31); + xtweak &= alphamask; + + /* next term */ + ret = vshlq_n_s32(inp, 1); + ret ^= xtweak; + + return ret; +} + +static void +armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule, + uint8x16_t *tweak, const uint8_t *from, uint8_t *to, int do_encrypt) +{ + uint8x16_t block; + + block = vld1q_u8(from) ^ *tweak; + + if (do_encrypt) + block = armv8_aes_enc(rounds - 1, key_schedule, block); + else + block = armv8_aes_dec(rounds - 1, key_schedule, block); + + vst1q_u8(to, block ^ *tweak); + + *tweak = vreinterpretq_u8_s32(xts_crank_lfsr(vreinterpretq_s32_u8(*tweak))); +} + +static void +armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule, + const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from, + uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt) +{ + uint8x16_t tweakreg; + uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); + size_t i, cnt; + + /* + * Prepare tweak as E_k2(IV). IV is specified as LE representation + * of a 64-bit block number which we allow to be passed in directly. + */ +#if BYTE_ORDER == LITTLE_ENDIAN + bcopy(iv, tweak, AES_XTS_IVSIZE); + /* Last 64 bits of IV are always zero. */ + bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); +#else +#error Only LITTLE_ENDIAN architectures are supported. +#endif + tweakreg = vld1q_u8(tweak); + tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg); + + cnt = len / AES_XTS_BLOCKSIZE; + for (i = 0; i < cnt; i++) { + armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg, + from, to, do_encrypt); + from += AES_XTS_BLOCKSIZE; + to += AES_XTS_BLOCKSIZE; + } +} + +void +armv8_aes_encrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[static AES_BLOCK_LEN]) +{ + + armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 1); +} + +void +armv8_aes_decrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[static AES_BLOCK_LEN]) +{ + + armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 0); +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202101071936.107JaL8B069216>