Date: Sat, 19 Nov 2011 16:30:07 +0000 (UTC) From: Rafal Jaworowski <raj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r227730 - in head/sys: arm/conf arm/mv boot/fdt/dts dev/cesa Message-ID: <201111191630.pAJGU7E0051272@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: raj Date: Sat Nov 19 16:30:06 2011 New Revision: 227730 URL: http://svn.freebsd.org/changeset/base/227730 Log: Initial version of cesa(4) driver for Marvell crypto engine and security accelerator. The following algorithms and schemes are supported: - 3DES, AES, DES - MD5, SHA1 Obtained from: Semihalf Written by: Piotr Ziecik Added: head/sys/dev/cesa/ head/sys/dev/cesa/cesa.c (contents, props changed) head/sys/dev/cesa/cesa.h (contents, props changed) Modified: head/sys/arm/conf/DB-88F6XXX head/sys/arm/conf/SHEEVAPLUG head/sys/arm/mv/files.mv head/sys/boot/fdt/dts/db88f6281.dts head/sys/boot/fdt/dts/sheevaplug.dts Modified: head/sys/arm/conf/DB-88F6XXX ============================================================================== --- head/sys/arm/conf/DB-88F6XXX Sat Nov 19 15:08:49 2011 (r227729) +++ head/sys/arm/conf/DB-88F6XXX Sat Nov 19 16:30:06 2011 (r227730) @@ -66,6 +66,10 @@ device mii device e1000phy device bpf +device cesa # Marvell security engine +device crypto +device cryptodev + # USB options USB_DEBUG # enable debug msgs device usb Modified: head/sys/arm/conf/SHEEVAPLUG ============================================================================== --- head/sys/arm/conf/SHEEVAPLUG Sat Nov 19 15:08:49 2011 (r227729) +++ head/sys/arm/conf/SHEEVAPLUG Sat Nov 19 16:30:06 2011 (r227730) @@ -60,6 +60,10 @@ options HZ=1000 options DEVICE_POLLING device vlan +device cesa # Marvell security engine +device crypto +device cryptodev + # USB options USB_DEBUG # enable debug msgs device usb Modified: head/sys/arm/mv/files.mv ============================================================================== --- head/sys/arm/mv/files.mv Sat Nov 19 15:08:49 2011 (r227729) +++ head/sys/arm/mv/files.mv Sat Nov 19 16:30:06 2011 (r227730) @@ -28,6 +28,7 @@ arm/mv/mv_sata.c optional ata | atamvsa arm/mv/timer.c standard arm/mv/twsi.c optional iicbus +dev/cesa/cesa.c optional cesa dev/mge/if_mge.c optional mge dev/mvs/mvs_soc.c optional mvs dev/uart/uart_dev_ns8250.c optional uart Modified: head/sys/boot/fdt/dts/db88f6281.dts ============================================================================== --- head/sys/boot/fdt/dts/db88f6281.dts Sat Nov 19 15:08:49 2011 (r227729) +++ head/sys/boot/fdt/dts/db88f6281.dts Sat Nov 19 16:30:06 2011 (r227730) @@ -239,6 +239,8 @@ reg = <0x30000 0x10000>; interrupts = <22>; interrupt-parent = <&PIC>; + + sram-handle = <&SRAM>; }; usb@50000 { Modified: head/sys/boot/fdt/dts/sheevaplug.dts ============================================================================== --- head/sys/boot/fdt/dts/sheevaplug.dts Sat Nov 19 15:08:49 2011 (r227729) +++ head/sys/boot/fdt/dts/sheevaplug.dts Sat Nov 19 16:30:06 2011 (r227730) @@ -236,6 +236,8 @@ reg = <0x30000 0x10000>; interrupts = <22>; interrupt-parent = <&PIC>; + + sram-handle = <&SRAM>; }; usb@50000 { Added: head/sys/dev/cesa/cesa.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/cesa/cesa.c Sat Nov 19 16:30:06 2011 (r227730) @@ -0,0 +1,1614 @@ +/*- + * Copyright (C) 2009-2011 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * CESA SRAM Memory Map: + * + * +------------------------+ <= sc->sc_sram_base + CESA_SRAM_SIZE + * | | + * | DATA | + * | | + * +------------------------+ <= sc->sc_sram_base + CESA_DATA(0) + * | struct cesa_sa_data | + * +------------------------+ + * | struct cesa_sa_hdesc | + * +------------------------+ <= sc->sc_sram_base + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/mbuf.h> +#include <sys/module.h> +#include <sys/mutex.h> +#include <sys/rman.h> + +#include <machine/bus.h> +#include <machine/intr.h> +#include <machine/resource.h> + +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include <sys/md5.h> +#include <crypto/sha1.h> +#include <crypto/rijndael/rijndael.h> +#include <opencrypto/cryptodev.h> +#include "cryptodev_if.h" + +#include <arm/mv/mvreg.h> +#include <arm/mv/mvwin.h> +#include <arm/mv/mvvar.h> +#include "cesa.h" + +#undef DEBUG + +static int cesa_probe(device_t); +static int cesa_attach(device_t); +static int cesa_detach(device_t); +static void cesa_intr(void *); +static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *); +static int cesa_freesession(device_t, u_int64_t); +static int cesa_process(device_t, struct cryptop *, int); + +static struct resource_spec cesa_res_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, + { -1, 0 } +}; + +static device_method_t cesa_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, cesa_probe), + DEVMETHOD(device_attach, cesa_attach), + DEVMETHOD(device_detach, cesa_detach), + + /* Bus interface */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD(bus_driver_added, bus_generic_driver_added), + + /* Crypto device methods */ + DEVMETHOD(cryptodev_newsession, cesa_newsession), + DEVMETHOD(cryptodev_freesession,cesa_freesession), + DEVMETHOD(cryptodev_process, cesa_process), + + { 0, 0 } +}; + +static driver_t cesa_driver = { + "cesa", + cesa_methods, + sizeof (struct cesa_softc) +}; +static devclass_t cesa_devclass; + +DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); +MODULE_DEPEND(cesa, crypto, 1, 1, 1); + +static void +cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) +{ +#ifdef DEBUG + device_t dev; + + dev = sc->sc_dev; + device_printf(dev, "CESA SA Hardware Descriptor:\n"); + device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); + device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); + device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); + device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); + device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); + device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); + device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); + device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); + device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); + device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); + device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); + device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); + device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); +#endif +} + +static void +cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct cesa_dma_mem *cdm; + + if (error) + return; + + KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); + cdm = arg; + cdm->cdm_paddr = segs->ds_addr; +} + +static int +cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, + bus_size_t size) +{ + int error; + + KASSERT(cdm->cdm_vaddr == NULL, + ("%s(): DMA memory descriptor in use.", __func__)); + + error = bus_dma_tag_create(NULL, /* parent */ + PAGE_SIZE, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filtfunc, filtfuncarg */ + size, 1, /* maxsize, nsegments */ + size, 0, /* maxsegsz, flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &cdm->cdm_tag); /* dmat */ + if (error) { + device_printf(sc->sc_dev, "failed to allocate busdma tag, error" + " %i!\n", error); + + goto err1; + } + + error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, + BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); + if (error) { + device_printf(sc->sc_dev, "failed to allocate DMA safe" + " memory, error %i!\n", error); + + goto err2; + } + + error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, + size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); + if (error) { + device_printf(sc->sc_dev, "cannot get address of the DMA" + " memory, error %i\n", error); + + goto err3; + } + + return (0); +err3: + bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); +err2: + bus_dma_tag_destroy(cdm->cdm_tag); +err1: + cdm->cdm_vaddr = NULL; + return (error); +} + +static void +cesa_free_dma_mem(struct cesa_dma_mem *cdm) +{ + + bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); + bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); + bus_dma_tag_destroy(cdm->cdm_tag); + cdm->cdm_vaddr = NULL; +} + +static void +cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) +{ + + /* Sync only if dma memory is valid */ + if (cdm->cdm_vaddr != NULL) + bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); +} + +static void +cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) +{ + + cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); + cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); + cesa_sync_dma_mem(&sc->sc_requests_cdm, op); +} + +static struct cesa_session * +cesa_alloc_session(struct cesa_softc *sc) +{ + struct cesa_session *cs; + + CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions); + + return (cs); +} + +static struct cesa_session * +cesa_get_session(struct cesa_softc *sc, uint32_t sid) +{ + + if (sid >= CESA_SESSIONS) + return (NULL); + + return (&sc->sc_sessions[sid]); +} + +static void +cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs) +{ + + CESA_GENERIC_FREE_LOCKED(sc, cs, sessions); +} + +static struct cesa_request * +cesa_alloc_request(struct cesa_softc *sc) +{ + struct cesa_request *cr; + + CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); + if (!cr) + return (NULL); + + STAILQ_INIT(&cr->cr_tdesc); + STAILQ_INIT(&cr->cr_sdesc); + + return (cr); +} + +static void +cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) +{ + + /* Free TDMA descriptors assigned to this request */ + CESA_LOCK(sc, tdesc); + STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); + CESA_UNLOCK(sc, tdesc); + + /* Free SA descriptors assigned to this request */ + CESA_LOCK(sc, sdesc); + STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); + CESA_UNLOCK(sc, sdesc); + + /* Unload DMA memory asociated with request */ + if (cr->cr_dmap_loaded) { + bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); + cr->cr_dmap_loaded = 0; + } + + CESA_GENERIC_FREE_LOCKED(sc, cr, requests); +} + +static void +cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) +{ + + CESA_LOCK(sc, requests); + STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); + CESA_UNLOCK(sc, requests); +} + +static struct cesa_tdma_desc * +cesa_alloc_tdesc(struct cesa_softc *sc) +{ + struct cesa_tdma_desc *ctd; + + CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); + + if (!ctd) + device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " + "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); + + return (ctd); +} + +static struct cesa_sa_desc * +cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) +{ + struct cesa_sa_desc *csd; + + CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); + if (!csd) { + device_printf(sc->sc_dev, "SA descriptors pool exhaused. " + "Consider increasing CESA_SA_DESCRIPTORS.\n"); + return (NULL); + } + + STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); + + /* Fill-in SA descriptor with default values */ + csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); + csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); + csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); + csd->csd_cshd->cshd_enc_src = 0; + csd->csd_cshd->cshd_enc_dst = 0; + csd->csd_cshd->cshd_enc_dlen = 0; + csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); + csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); + csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); + csd->csd_cshd->cshd_mac_src = 0; + csd->csd_cshd->cshd_mac_dlen = 0; + + return (csd); +} + +static struct cesa_tdma_desc * +cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, + bus_size_t size) +{ + struct cesa_tdma_desc *ctd; + + ctd = cesa_alloc_tdesc(sc); + if (!ctd) + return (NULL); + + ctd->ctd_cthd->cthd_dst = dst; + ctd->ctd_cthd->cthd_src = src; + ctd->ctd_cthd->cthd_byte_count = size; + + /* Handle special control packet */ + if (size != 0) + ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; + else + ctd->ctd_cthd->cthd_flags = 0; + + return (ctd); +} + +static struct cesa_tdma_desc * +cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) +{ + + return (cesa_tdma_copy(sc, sc->sc_sram_base + + sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, + sizeof(struct cesa_sa_data))); +} + +static struct cesa_tdma_desc * +cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) +{ + + return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base + + sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); +} + +static struct cesa_tdma_desc * +cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) +{ + + return (cesa_tdma_copy(sc, sc->sc_sram_base, csd->csd_cshd_paddr, + sizeof(struct cesa_sa_hdesc))); +} + +static void +cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) +{ + struct cesa_tdma_desc *ctd_prev; + + if (!STAILQ_EMPTY(&cr->cr_tdesc)) { + ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); + ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; + } + + ctd->ctd_cthd->cthd_next = 0; + STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); +} + +static int +cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, + struct cesa_packet *cp, struct cesa_sa_desc *csd) +{ + struct cesa_tdma_desc *ctd, *tmp; + + /* Copy SA descriptor for this packet */ + ctd = cesa_tdma_copy_sdesc(sc, csd); + if (!ctd) + return (ENOMEM); + + cesa_append_tdesc(cr, ctd); + + /* Copy data to be processed */ + STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) + cesa_append_tdesc(cr, ctd); + STAILQ_INIT(&cp->cp_copyin); + + /* Insert control descriptor */ + ctd = cesa_tdma_copy(sc, 0, 0, 0); + if (!ctd) + return (ENOMEM); + + cesa_append_tdesc(cr, ctd); + + /* Copy back results */ + STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) + cesa_append_tdesc(cr, ctd); + STAILQ_INIT(&cp->cp_copyout); + + return (0); +} + +static int +cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) +{ + uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; + uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; + SHA1_CTX sha1ctx; + MD5_CTX md5ctx; + uint32_t *hout; + uint32_t *hin; + int i; + + memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); + memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); + for (i = 0; i < mklen; i++) { + ipad[i] ^= mkey[i]; + opad[i] ^= mkey[i]; + } + + hin = (uint32_t *)cs->cs_hiv_in; + hout = (uint32_t *)cs->cs_hiv_out; + + switch (alg) { + case CRYPTO_MD5_HMAC: + MD5Init(&md5ctx); + MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN); + memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); + MD5Init(&md5ctx); + MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN); + memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); + break; + case CRYPTO_SHA1_HMAC: + SHA1Init(&sha1ctx); + SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN); + memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); + SHA1Init(&sha1ctx); + SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN); + memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); + break; + default: + return (EINVAL); + } + + for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { + hin[i] = htobe32(hin[i]); + hout[i] = htobe32(hout[i]); + } + + return (0); +} + +static int +cesa_prep_aes_key(struct cesa_session *cs) +{ + uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; + uint32_t *dkey; + int i; + + rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); + + cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; + dkey = (uint32_t *)cs->cs_aes_dkey; + + switch (cs->cs_klen) { + case 16: + cs->cs_config |= CESA_CSH_AES_KLEN_128; + for (i = 0; i < 4; i++) + *dkey++ = htobe32(ek[4 * 10 + i]); + break; + case 24: + cs->cs_config |= CESA_CSH_AES_KLEN_192; + for (i = 0; i < 4; i++) + *dkey++ = htobe32(ek[4 * 12 + i]); + for (i = 0; i < 2; i++) + *dkey++ = htobe32(ek[4 * 11 + 2 + i]); + break; + case 32: + cs->cs_config |= CESA_CSH_AES_KLEN_256; + for (i = 0; i < 4; i++) + *dkey++ = htobe32(ek[4 * 14 + i]); + for (i = 0; i < 4; i++) + *dkey++ = htobe32(ek[4 * 13 + i]); + break; + default: + return (EINVAL); + } + + return (0); +} + +static int +cesa_is_hash(int alg) +{ + + switch (alg) { + case CRYPTO_MD5: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + return (1); + default: + return (0); + } +} + +static void +cesa_start_packet(struct cesa_packet *cp, unsigned int size) +{ + + cp->cp_size = size; + cp->cp_offset = 0; + STAILQ_INIT(&cp->cp_copyin); + STAILQ_INIT(&cp->cp_copyout); +} + +static int +cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, + bus_dma_segment_t *seg) +{ + struct cesa_tdma_desc *ctd; + unsigned int bsize; + + /* Calculate size of block copy */ + bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); + + if (bsize > 0) { + ctd = cesa_tdma_copy(sc, sc->sc_sram_base + + CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); + if (!ctd) + return (-ENOMEM); + + STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); + + ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base + + CESA_DATA(cp->cp_offset), bsize); + if (!ctd) + return (-ENOMEM); + + STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); + + seg->ds_len -= bsize; + seg->ds_addr += bsize; + cp->cp_offset += bsize; + } + + return (bsize); +} + +static void +cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + unsigned int mpsize, fragmented; + unsigned int mlen, mskip, tmlen; + struct cesa_chain_info *cci; + unsigned int elen, eskip; + unsigned int skip, len; + struct cesa_sa_desc *csd; + struct cesa_request *cr; + struct cesa_softc *sc; + struct cesa_packet cp; + bus_dma_segment_t seg; + uint32_t config; + int size; + + cci = arg; + sc = cci->cci_sc; + cr = cci->cci_cr; + + if (error) { + cci->cci_error = error; + return; + } + + elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; + eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; + mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; + mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; + + if (elen && mlen && + ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || + (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || + (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { + /* + * Data alignment in the request does not meet CESA requiremnts + * for combined encryption/decryption and hashing. We have to + * split the request to separate operations and process them + * one by one. + */ + config = cci->cci_config; + if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { + config &= ~CESA_CSHD_OP_MASK; + + cci->cci_config = config | CESA_CSHD_MAC; + cci->cci_enc = NULL; + cci->cci_mac = cr->cr_mac; + cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + + cci->cci_config = config | CESA_CSHD_ENC; + cci->cci_enc = cr->cr_enc; + cci->cci_mac = NULL; + cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + } else { + config &= ~CESA_CSHD_OP_MASK; + + cci->cci_config = config | CESA_CSHD_ENC; + cci->cci_enc = cr->cr_enc; + cci->cci_mac = NULL; + cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + + cci->cci_config = config | CESA_CSHD_MAC; + cci->cci_enc = NULL; + cci->cci_mac = cr->cr_mac; + cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); + } + + return; + } + + tmlen = mlen; + fragmented = 0; + mpsize = CESA_MAX_PACKET_SIZE; + mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); + + if (elen && mlen) { + skip = MIN(eskip, mskip); + len = MAX(elen + eskip, mlen + mskip) - skip; + } else if (elen) { + skip = eskip; + len = elen; + } else { + skip = mskip; + len = mlen; + } + + /* Start first packet in chain */ + cesa_start_packet(&cp, MIN(mpsize, len)); + + while (nseg-- && len > 0) { + seg = *(segs++); + + /* + * Skip data in buffer on which neither ENC nor MAC operation + * is requested. + */ + if (skip > 0) { + size = MIN(skip, seg.ds_len); + skip -= size; + + seg.ds_addr += size; + seg.ds_len -= size; + + if (eskip > 0) + eskip -= size; + + if (mskip > 0) + mskip -= size; + + if (seg.ds_len == 0) + continue; + } + + while (1) { + /* + * Fill in current packet with data. Break if there is + * no more data in current DMA segment or an error + * occured. + */ + size = cesa_fill_packet(sc, &cp, &seg); + if (size <= 0) { + error = -size; + break; + } + + len -= size; + + /* If packet is full, append it to the chain */ + if (cp.cp_size == cp.cp_offset) { + csd = cesa_alloc_sdesc(sc, cr); + if (!csd) { + error = ENOMEM; + break; + } + + /* Create SA descriptor for this packet */ + csd->csd_cshd->cshd_config = cci->cci_config; + csd->csd_cshd->cshd_mac_total_dlen = tmlen; + + /* + * Enable fragmentation if request will not fit + * into one packet. + */ + if (len > 0) { + if (!fragmented) { + fragmented = 1; + csd->csd_cshd->cshd_config |= + CESA_CSHD_FRAG_FIRST; + } else + csd->csd_cshd->cshd_config |= + CESA_CSHD_FRAG_MIDDLE; + } else if (fragmented) + csd->csd_cshd->cshd_config |= + CESA_CSHD_FRAG_LAST; + + if (eskip < cp.cp_size && elen > 0) { + csd->csd_cshd->cshd_enc_src = + CESA_DATA(eskip); + csd->csd_cshd->cshd_enc_dst = + CESA_DATA(eskip); + csd->csd_cshd->cshd_enc_dlen = + MIN(elen, cp.cp_size - eskip); + } + + if (mskip < cp.cp_size && mlen > 0) { + csd->csd_cshd->cshd_mac_src = + CESA_DATA(mskip); + csd->csd_cshd->cshd_mac_dlen = + MIN(mlen, cp.cp_size - mskip); + } + + elen -= csd->csd_cshd->cshd_enc_dlen; + eskip -= MIN(eskip, cp.cp_size); + mlen -= csd->csd_cshd->cshd_mac_dlen; + mskip -= MIN(mskip, cp.cp_size); + + cesa_dump_cshd(sc, csd->csd_cshd); + + /* Append packet to the request */ + error = cesa_append_packet(sc, cr, &cp, csd); + if (error) + break; + + /* Start a new packet, as current is full */ + cesa_start_packet(&cp, MIN(mpsize, len)); + } + } + + if (error) + break; + } + + if (error) { + /* + * Move all allocated resources to the request. They will be + * freed later. + */ + STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); + STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); + cci->cci_error = error; + } +} + +static void +cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, + bus_size_t size, int error) +{ + + cesa_create_chain_cb(arg, segs, nseg, error); +} + +static int +cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) +{ + struct cesa_chain_info cci; + struct cesa_tdma_desc *ctd; + uint32_t config; + int error; + + error = 0; + CESA_LOCK_ASSERT(sc, sessions); + + /* Create request metadata */ + if (cr->cr_enc) { + if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && + (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) + memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, + cr->cr_cs->cs_klen); + else + memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, + cr->cr_cs->cs_klen); + } + + if (cr->cr_mac) { + memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, + CESA_MAX_HASH_LEN); + memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, + CESA_MAX_HASH_LEN); + } + + ctd = cesa_tdma_copyin_sa_data(sc, cr); + if (!ctd) + return (ENOMEM); + + cesa_append_tdesc(cr, ctd); + + /* Prepare SA configuration */ + config = cr->cr_cs->cs_config; + + if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) + config |= CESA_CSHD_DECRYPT; + if (cr->cr_enc && !cr->cr_mac) + config |= CESA_CSHD_ENC; + if (!cr->cr_enc && cr->cr_mac) + config |= CESA_CSHD_MAC; + if (cr->cr_enc && cr->cr_mac) + config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : + CESA_CSHD_ENC_AND_MAC; + + /* Create data packets */ + cci.cci_sc = sc; + cci.cci_cr = cr; + cci.cci_enc = cr->cr_enc; + cci.cci_mac = cr->cr_mac; + cci.cci_config = config; + cci.cci_error = 0; + + if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) + error = bus_dmamap_load_uio(sc->sc_data_dtag, + cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, + cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); + else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) + error = bus_dmamap_load_mbuf(sc->sc_data_dtag, + cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, + cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); + else + error = bus_dmamap_load(sc->sc_data_dtag, + cr->cr_dmap, cr->cr_crp->crp_buf, + cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, + BUS_DMA_NOWAIT); + + if (!error) + cr->cr_dmap_loaded = 1; + + if (cci.cci_error) + error = cci.cci_error; + + if (error) + return (error); + + /* Read back request metadata */ + ctd = cesa_tdma_copyout_sa_data(sc, cr); + if (!ctd) + return (ENOMEM); + + cesa_append_tdesc(cr, ctd); + + return (0); +} + +static void +cesa_execute(struct cesa_softc *sc) +{ + struct cesa_tdma_desc *prev_ctd, *ctd; + struct cesa_request *prev_cr, *cr; + + CESA_LOCK(sc, requests); + + /* + * If ready list is empty, there is nothing to execute. If queued list + * is not empty, the hardware is busy and we cannot start another + * execution. + */ + if (STAILQ_EMPTY(&sc->sc_ready_requests) || + !STAILQ_EMPTY(&sc->sc_queued_requests)) { + CESA_UNLOCK(sc, requests); + return; + } + + /* Move all ready requests to queued list */ + STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); + STAILQ_INIT(&sc->sc_ready_requests); + + /* Create one execution chain from all requests on the list */ + if (STAILQ_FIRST(&sc->sc_queued_requests) != + STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { + prev_cr = NULL; + cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | + BUS_DMASYNC_POSTWRITE); + + STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { + if (prev_cr) { + ctd = STAILQ_FIRST(&cr->cr_tdesc); + prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, + cesa_tdma_desc, ctd_stq); + + prev_ctd->ctd_cthd->cthd_next = + ctd->ctd_cthd_paddr; + } + + prev_cr = cr; + } + + cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201111191630.pAJGU7E0051272>