From owner-svn-src-head@FreeBSD.ORG Thu Sep 23 11:57:25 2010 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 600FE106566B; Thu, 23 Sep 2010 11:57:25 +0000 (UTC) (envelope-from pjd@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 4E9EC8FC08; Thu, 23 Sep 2010 11:57:25 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o8NBvPw0068298; Thu, 23 Sep 2010 11:57:25 GMT (envelope-from pjd@svn.freebsd.org) Received: (from pjd@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o8NBvPwv068294; Thu, 23 Sep 2010 11:57:25 GMT (envelope-from pjd@svn.freebsd.org) Message-Id: <201009231157.o8NBvPwv068294@svn.freebsd.org> From: Pawel Jakub Dawidek Date: Thu, 23 Sep 2010 11:57:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r213069 - head/sys/crypto/aesni X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 23 Sep 2010 11:57:25 -0000 Author: pjd Date: Thu Sep 23 11:57:25 2010 New Revision: 213069 URL: http://svn.freebsd.org/changeset/base/213069 Log: Add support for CRYPTO_AES_XTS. MFC after: 1 week Modified: head/sys/crypto/aesni/aesni.c head/sys/crypto/aesni/aesni.h head/sys/crypto/aesni/aesni_wrap.c Modified: head/sys/crypto/aesni/aesni.c ============================================================================== --- head/sys/crypto/aesni/aesni.c Thu Sep 23 11:52:32 2010 (r213068) +++ head/sys/crypto/aesni/aesni.c Thu Sep 23 11:57:25 2010 (r213069) @@ -74,7 +74,7 @@ aesni_probe(device_t dev) device_printf(dev, "No AESNI support.\n"); return (EINVAL); } - device_set_desc_copy(dev, "AES-CBC"); + device_set_desc_copy(dev, "AES-CBC,AES-XTS"); return (0); } @@ -94,6 +94,7 @@ aesni_attach(device_t dev) rw_init(&sc->lock, "aesni_lock"); crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); return (0); } @@ -140,6 +141,7 @@ aesni_newsession(device_t dev, uint32_t for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_XTS: if (encini != NULL) return (EINVAL); encini = cri; @@ -172,6 +174,7 @@ aesni_newsession(device_t dev, uint32_t ses->used = 1; TAILQ_INSERT_TAIL(&sc->sessions, ses, next); rw_wunlock(&sc->lock); + ses->algo = encini->cri_alg; error = aesni_cipher_setup(ses, encini); if (error != 0) { @@ -243,6 +246,7 @@ aesni_process(device_t dev, struct crypt for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { case CRYPTO_AES_CBC: + case CRYPTO_AES_XTS: if (enccrd != NULL) { error = EINVAL; goto out; Modified: head/sys/crypto/aesni/aesni.h ============================================================================== --- head/sys/crypto/aesni/aesni.h Thu Sep 23 11:52:32 2010 (r213068) +++ head/sys/crypto/aesni/aesni.h Thu Sep 23 11:57:25 2010 (r213069) @@ -55,7 +55,9 @@ struct aesni_session { uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16); uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16); + uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16); uint8_t iv[AES_BLOCK_LEN]; + int algo; int rounds; /* uint8_t *ses_ictx; */ /* uint8_t *ses_octx; */ Modified: head/sys/crypto/aesni/aesni_wrap.c ============================================================================== --- head/sys/crypto/aesni/aesni_wrap.c Thu Sep 23 11:52:32 2010 (r213068) +++ head/sys/crypto/aesni/aesni_wrap.c Thu Sep 23 11:57:25 2010 (r213069) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2010 Konstantin Belousov + * Copyright (c) 2010 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -105,28 +106,138 @@ aesni_decrypt_ecb(int rounds, const void } } +#define AES_XTS_BLOCKSIZE 16 +#define AES_XTS_IVSIZE 8 +#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ + +static void +aesni_crypt_xts_block(int rounds, const void *key_schedule, uint8_t *tweak, + const uint8_t *from, uint8_t *to, int do_encrypt) +{ + uint8_t block[AES_XTS_BLOCKSIZE]; + u_int i, carry_in, carry_out; + + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + block[i] = from[i] ^ tweak[i]; + + if (do_encrypt) + aesni_enc(rounds - 1, key_schedule, block, to, NULL); + else + aesni_dec(rounds - 1, key_schedule, block, to, NULL); + + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + to[i] ^= tweak[i]; + + /* Exponentiate tweak. */ + carry_in = 0; + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { + carry_out = tweak[i] & 0x80; + tweak[i] = (tweak[i] << 1) | (carry_in ? 1 : 0); + carry_in = carry_out; + } + if (carry_in) + tweak[0] ^= AES_XTS_ALPHA; + bzero(block, sizeof(block)); +} + +static void +aesni_crypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) +{ + uint8_t tweak[AES_XTS_BLOCKSIZE]; + uint64_t blocknum; + size_t i; + + /* + * Prepare tweak as E_k2(IV). IV is specified as LE representation + * of a 64-bit block number which we allow to be passed in directly. + */ + bcopy(iv, &blocknum, AES_XTS_IVSIZE); + for (i = 0; i < AES_XTS_IVSIZE; i++) { + tweak[i] = blocknum & 0xff; + blocknum >>= 8; + } + /* Last 64 bits of IV are always zero. */ + bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); + aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL); + + len /= AES_XTS_BLOCKSIZE; + for (i = 0; i < len; i++) { + aesni_crypt_xts_block(rounds, data_schedule, tweak, from, to, + do_encrypt); + from += AES_XTS_BLOCKSIZE; + to += AES_XTS_BLOCKSIZE; + } + + bzero(tweak, sizeof(tweak)); +} + +static void +aesni_encrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN]) +{ + + aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 1); +} + +static void +aesni_decrypt_xts(int rounds, const void *data_schedule, + const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, + const uint8_t iv[AES_BLOCK_LEN]) +{ + + aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, + iv, 0); +} + static int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, int keylen) { - switch (keylen) { - case 128: - ses->rounds = AES128_ROUNDS; - break; - case 192: - ses->rounds = AES192_ROUNDS; + switch (ses->algo) { + case CRYPTO_AES_CBC: + switch (keylen) { + case 128: + ses->rounds = AES128_ROUNDS; + break; + case 192: + ses->rounds = AES192_ROUNDS; + break; + case 256: + ses->rounds = AES256_ROUNDS; + break; + default: + return (EINVAL); + } break; - case 256: - ses->rounds = AES256_ROUNDS; + case CRYPTO_AES_XTS: + switch (keylen) { + case 256: + ses->rounds = AES128_ROUNDS; + break; + case 512: + ses->rounds = AES256_ROUNDS; + break; + default: + return (EINVAL); + } break; default: return (EINVAL); } - + aesni_set_enckey(key, ses->enc_schedule, ses->rounds); aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds); - arc4rand(ses->iv, sizeof(ses->iv), 0); + if (ses->algo == CRYPTO_AES_XTS) + arc4rand(ses->iv, sizeof(ses->iv), 0); + else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_set_enckey(key + keylen / 16, ses->xts_schedule, + ses->rounds); + } return (0); } @@ -137,14 +248,14 @@ aesni_cipher_setup(struct aesni_session struct thread *td; int error; - td = curthread; - error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); - if (error == 0) { + td = curthread; + error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); + if (error == 0) { error = aesni_cipher_setup_common(ses, encini->cri_key, encini->cri_klen); - fpu_kern_leave(td, &ses->fpu_ctx); - } - return (error); + fpu_kern_leave(td, &ses->fpu_ctx); + } + return (error); } int @@ -163,7 +274,7 @@ aesni_cipher_process(struct aesni_sessio error = fpu_kern_enter(td, &ses->fpu_ctx, FPU_KERN_NORMAL); if (error != 0) goto out; - + if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { error = aesni_cipher_setup_common(ses, enccrd->crd_key, enccrd->crd_klen); @@ -174,21 +285,31 @@ aesni_cipher_process(struct aesni_sessio if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - - aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, - enccrd->crd_len, buf, buf, ses->iv); + if (ses->algo == CRYPTO_AES_CBC) { + aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, + enccrd->crd_len, buf, buf, ses->iv); + } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_encrypt_xts(ses->rounds, ses->enc_schedule, + ses->xts_schedule, enccrd->crd_len, buf, buf, + ses->iv); + } } else { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); else crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, - enccrd->crd_len, buf, ses->iv); + if (ses->algo == CRYPTO_AES_CBC) { + aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, + enccrd->crd_len, buf, ses->iv); + } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + aesni_decrypt_xts(ses->rounds, ses->enc_schedule, + ses->xts_schedule, enccrd->crd_len, buf, buf, + ses->iv); + } } fpu_kern_leave(td, &ses->fpu_ctx); if (allocated)