Date: Mon, 8 Sep 2014 04:50:10 GMT From: John-Mark Gurney <jmg@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 1199105 for review Message-ID: <201409080450.s884oA0v023829@skunkworks.freebsd.org>
index | next in thread | raw e-mail
http://p4web.freebsd.org/@@1199105?ac=10 Change 1199105 by jmg@jmg_carbon2 on 2014/08/23 00:03:21 add support for ICM to aesni driver... properly initalize error when authbuf fails to allocate #if 0 out debug function... Sponsored by: FreeBSD Foundation Affected files ... .. //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#4 edit .. //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#3 edit .. //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#3 edit Differences ... ==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.c#4 (text+ko) ==== @@ -110,11 +110,12 @@ rw_init(&sc->lock, "aesni_lock"); crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); return (0); } @@ -164,6 +165,7 @@ for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: case CRYPTO_AES_NIST_GCM_16: if (encini != NULL) { @@ -294,6 +296,7 @@ for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: if (enccrd != NULL) { error = EINVAL; @@ -433,6 +436,7 @@ return (error); } +#if 0 static void printhexstr(uint8_t *ptr, int len) { @@ -441,6 +445,7 @@ for (i = 0; i < len; i++) printf("%02hhx", ptr[i]); } +#endif /* * authcrd contains the associated date. @@ -458,6 +463,10 @@ encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; + if (enccrd->crd_alg == CRYPTO_AES_ICM && + (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) + return (EINVAL); + buf = aesni_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) return (ENOMEM); @@ -466,8 +475,10 @@ authallocated = 0; if (authcrd != NULL) { authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); - if (authbuf == NULL) + if (authbuf == NULL) { + error = ENOMEM; goto out1; + } } td = curthread; @@ -486,6 +497,7 @@ /* XXX - validate that enccrd and authcrd have/use same key? */ switch (enccrd->crd_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: ivlen = 16; break; case CRYPTO_AES_XTS: @@ -535,6 +547,10 @@ aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, enccrd->crd_len, buf, ses->iv); break; + case CRYPTO_AES_ICM: + aesni_encrypt_icm(ses->rounds, ses->enc_schedule, + enccrd->crd_len, buf, buf, ses->iv); + break; case CRYPTO_AES_XTS: if (encflag) aesni_encrypt_xts(ses->rounds, ses->enc_schedule, ==== //depot/projects/opencrypto/sys/crypto/aesni/aesni.h#3 (text+ko) ==== @@ -88,6 +88,9 @@ size_t len, const uint8_t *from, uint8_t *to); void aesni_decrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to); +void aesni_encrypt_icm(int rounds, const void *key_schedule /*__aligned(16)*/, + size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN]); void aesni_encrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/, const void *tweak_schedule /*__aligned(16)*/, size_t len, ==== //depot/projects/opencrypto/sys/crypto/aesni/aesni_wrap.c#3 (text+ko) ==== @@ -44,6 +44,7 @@ #include <opencrypto/gmac.h> #include "aesencdec.h" +#include <smmintrin.h> MALLOC_DECLARE(M_AESNI); @@ -182,6 +183,104 @@ } } +/* + * mixed endian increment, low 64bits stored in hi word to be compatible + * with _icm's BSWAP. + */ +static inline __m128i +nextc(__m128i x) +{ + const __m128i ONE = _mm_setr_epi32(0, 0, 1, 0); + const __m128i ZERO = _mm_setzero_si128(); + + x = _mm_add_epi64(x, ONE); + __m128i t = _mm_cmpeq_epi64(x, ZERO); + t = _mm_unpackhi_epi64(t, ZERO); + x = _mm_sub_epi64(x, t); + + return x; +} + +void +aesni_encrypt_icm(int rounds, const void *key_schedule, size_t len, + const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN]) +{ + __m128i tot; + __m128i tmp1, tmp2, tmp3, tmp4; + __m128i tmp5, tmp6, tmp7, tmp8; + __m128i ctr1, ctr2, ctr3, ctr4; + __m128i ctr5, ctr6, ctr7, ctr8; + __m128i BSWAP_EPI64; + __m128i tout[8]; + struct blocks8 *top; + const struct blocks8 *blks; + size_t i, cnt; + + BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7); + + ctr1 = _mm_loadu_si128((__m128i*)iv); + ctr1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64); + + cnt = len / AES_BLOCK_LEN / 8; + for (i = 0; i < cnt; i++) { + tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64); + ctr2 = nextc(ctr1); + tmp2 = _mm_shuffle_epi8(ctr2, BSWAP_EPI64); + ctr3 = nextc(ctr2); + tmp3 = _mm_shuffle_epi8(ctr3, BSWAP_EPI64); + ctr4 = nextc(ctr3); + tmp4 = _mm_shuffle_epi8(ctr4, BSWAP_EPI64); + ctr5 = nextc(ctr4); + tmp5 = _mm_shuffle_epi8(ctr5, BSWAP_EPI64); + ctr6 = nextc(ctr5); + tmp6 = _mm_shuffle_epi8(ctr6, BSWAP_EPI64); + ctr7 = nextc(ctr6); + tmp7 = _mm_shuffle_epi8(ctr7, BSWAP_EPI64); + ctr8 = nextc(ctr7); + tmp8 = _mm_shuffle_epi8(ctr8, BSWAP_EPI64); + ctr1 = nextc(ctr8); + + blks = (const struct blocks8 *)from; + top = (struct blocks8 *)to; + aesni_enc8(rounds - 1, key_schedule, tmp1, tmp2, tmp3, tmp4, + tmp5, tmp6, tmp7, tmp8, tout); + + top->blk[0] = blks->blk[0] ^ tout[0]; + top->blk[1] = blks->blk[1] ^ tout[1]; + top->blk[2] = blks->blk[2] ^ tout[2]; + top->blk[3] = blks->blk[3] ^ tout[3]; + top->blk[4] = blks->blk[4] ^ tout[4]; + top->blk[5] = blks->blk[5] ^ tout[5]; + top->blk[6] = blks->blk[6] ^ tout[6]; + top->blk[7] = blks->blk[7] ^ tout[7]; + + from += AES_BLOCK_LEN * 8; + to += AES_BLOCK_LEN * 8; + } + i *= 8; + cnt = len / AES_BLOCK_LEN; + for (; i < cnt; i++) { + tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64); + ctr1 = nextc(ctr1); + + tot = aesni_enc(rounds - 1, key_schedule, tmp1); + + tot = tot ^ _mm_loadu_si128((const __m128i *)from); + _mm_storeu_si128((__m128i *)to, tot); + + from += AES_BLOCK_LEN; + to += AES_BLOCK_LEN; + } + + /* handle remaining partial round */ + if (len % AES_BLOCK_LEN != 0) { + tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64); + tot = aesni_enc(rounds - 1, key_schedule, tmp1); + tot = tot ^ _mm_loadu_si128((const __m128i *)from); + memcpy(to, &tot, len % AES_BLOCK_LEN); + } +} + #define AES_XTS_BLOCKSIZE 16 #define AES_XTS_IVSIZE 8 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ @@ -342,6 +441,7 @@ switch (ses->algo) { case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: case CRYPTO_AES_NIST_GCM_16: switch (keylen) { case 128: @@ -354,7 +454,7 @@ ses->rounds = AES256_ROUNDS; break; default: - CRYPTDEB("invalid CBC/GCM key length"); + CRYPTDEB("invalid CBC/ICM/GCM key length"); return (EINVAL); } break; @@ -381,10 +481,9 @@ /* setup IV */ if (ses->algo == CRYPTO_AES_CBC || ses->algo == CRYPTO_AES_NIST_GCM_16) arc4rand(ses->iv, sizeof(ses->iv), 0); - else /* if (ses->algo == CRYPTO_AES_XTS) */ { + else if (ses->algo == CRYPTO_AES_XTS) aesni_set_enckey(key + keylen / 16, ses->xts_schedule, ses->rounds); - } return (0); }help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201409080450.s884oA0v023829>