diff options
Diffstat (limited to 'drivers/crypto/ace.c')
-rw-r--r-- | drivers/crypto/ace.c | 2651 |
1 files changed, 0 insertions, 2651 deletions
diff --git a/drivers/crypto/ace.c b/drivers/crypto/ace.c deleted file mode 100644 index 21ddf96..0000000 --- a/drivers/crypto/ace.c +++ /dev/null @@ -1,2651 +0,0 @@ -/* - * Cryptographic API. - * - * Support for ACE (Advanced Crypto Engine) for S5PV210/EXYNOS4210. - * - * Copyright (c) 2011 Samsung Electronics - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/err.h> -#include <linux/scatterlist.h> -#include <linux/platform_device.h> -#include <linux/clk.h> -#include <linux/interrupt.h> -#include <linux/dma-mapping.h> -#include <linux/memory.h> -#include <linux/delay.h> -#include <linux/version.h> -#include <linux/hrtimer.h> - -#include <asm/cacheflush.h> - -#include <crypto/aes.h> -#include <crypto/internal/hash.h> -#include <crypto/sha.h> -#include <crypto/scatterwalk.h> - -#include <mach/secmem.h> - -#include "ace.h" -#include "ace_sfr.h" - -#define S5P_ACE_DRIVER_NAME "s5p-ace" -#define ACE_AES_MIN_BLOCK_SIZE 16 - -#undef ACE_USE_ACP -#ifdef ACE_USE_ACP -#define PA_SSS_USER_CON 0x10010344 -#define ACE_ARCACHE 0xA -#define ACE_AWCACHE 0xA -#endif - -#undef ACE_DEBUG_HEARTBEAT -#undef ACE_DEBUG_WATCHDOG - -#ifdef CONFIG_ACE_DEBUG -#define S5P_ACE_DEBUG(args...) printk(KERN_INFO args) -#else -#define S5P_ACE_DEBUG(args...) -#endif - -#define s5p_ace_read_sfr(_sfr_) __raw_readl(s5p_ace_dev.ace_base + (_sfr_)) -#define s5p_ace_write_sfr(_sfr_, _val_) __raw_writel((_val_), s5p_ace_dev.ace_base + (_sfr_)) - -enum s5p_cpu_type { - TYPE_S5PV210, - TYPE_EXYNOS, -}; - -enum { - FLAGS_BC_BUSY, - FLAGS_HASH_BUSY, - FLAGS_SUSPENDED, - FLAGS_USE_SW -}; - -static struct s5p_ace_device s5p_ace_dev; - -#ifdef CONFIG_ACE_BC_ASYNC -static void s5p_ace_bc_task(unsigned long data); -#endif - -#define ACE_CLOCK_ON 0 -#define ACE_CLOCK_OFF 1 - -static int count_clk; -static int count_clk_delta; - -static int count_use_sw; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) -#define ACE_HEARTBEAT_MS 10000 -#define ACE_WATCHDOG_MS 500 - -struct timeval timestamp_base; -struct timeval timestamp[5]; - -static inline void s5p_ace_dump(void) -{ - int i; - char *str[] = {"request: ", "dma start: ", "dma end: ", "suspend: ", "resume: "}; - - for (i = 0; i < 5; i++) - printk(KERN_INFO "%s%5lu.%06lu\n", - str[i], timestamp[i].tv_sec - timestamp_base.tv_sec, timestamp[i].tv_usec); - printk(KERN_INFO "clock: [%d - %d]\n", count_clk, count_clk_delta); -} -#endif - -struct s5p_ace_reqctx { - u32 mode; -}; - -struct s5p_ace_device { - void __iomem *ace_base; - struct clk *clock; -#if defined(CONFIG_ACE_BC_IRQMODE) || defined(CONFIG_ACE_HASH_IRQMODE) - int irq; -#endif -#ifdef ACE_USE_ACP - void __iomem *sss_usercon; -#endif - spinlock_t lock; - unsigned long flags; - - struct hrtimer timer; - struct work_struct work; -#ifdef ACE_DEBUG_HEARTBEAT - struct hrtimer heartbeat; -#endif -#ifdef ACE_DEBUG_WATCHDOG - struct hrtimer watchdog_bc; -#endif - -#ifdef CONFIG_ACE_BC_ASYNC - struct crypto_queue queue_bc; - struct tasklet_struct task_bc; - int rc_depth_bc; -#endif - - struct s5p_ace_aes_ctx *ctx_bc; - -#ifdef CONFIG_ACE_HASH_ASYNC - struct crypto_queue queue_hash; - struct tasklet_struct task_hash; -#endif - enum s5p_cpu_type cputype; -}; - -#if defined(CONFIG_ACE_HASH_SHA1) || defined(CONFIG_ACE_HASH_SHA256) -struct crypto_shash *sw_tfm; -struct crypto_hash **fallback_hash; -#endif -struct secmem_crypto_driver_ftn secmem_ftn; - -static void s5p_ace_init_clock_gating(void) -{ - count_clk = 0; - count_clk_delta = 0; -} - -static void s5p_ace_deferred_clock_disable(struct work_struct *work) -{ - unsigned long flags; - int tmp; - - if (count_clk_delta == 0) - return; - - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - count_clk -= count_clk_delta; - count_clk_delta = 0; - tmp = count_clk; - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - if (tmp == 0) { - clk_disable(s5p_ace_dev.clock); - S5P_ACE_DEBUG("ACE clock OFF\n"); - } -} - -static enum hrtimer_restart s5p_ace_timer_func(struct hrtimer *timer) -{ - S5P_ACE_DEBUG("ACE HRTIMER\n"); - - /* It seems that "schedule_work" is expensive. */ - schedule_work(&s5p_ace_dev.work); - - return HRTIMER_NORESTART; -} - -static void s5p_ace_clock_gating(int status) -{ - unsigned long flags; - int tmp; - - if (status == ACE_CLOCK_ON) { - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - tmp = count_clk++; - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - if (tmp == 0) { - clk_enable(s5p_ace_dev.clock); - S5P_ACE_DEBUG("ACE clock ON\n"); - } - } else if (status == ACE_CLOCK_OFF) { - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - if (count_clk > 1) - count_clk--; - else - count_clk_delta++; - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - hrtimer_start(&s5p_ace_dev.timer, - ns_to_ktime((u64)500 * NSEC_PER_MSEC), - HRTIMER_MODE_REL); - } -} - -struct s5p_ace_aes_ctx { - u32 keylen; - - u32 sfr_ctrl; - u8 sfr_key[AES_MAX_KEY_SIZE]; - u8 sfr_semikey[AES_BLOCK_SIZE]; - - struct crypto_blkcipher *fallback_bc; -#ifdef CONFIG_ACE_BC_ASYNC - struct ablkcipher_request *req; - struct crypto_ablkcipher *fallback_abc; - struct crypto_tfm *origin_tfm; -#else - struct crypto_blkcipher *origin_tfm; - -#endif - size_t total; - struct scatterlist *in_sg; - size_t in_ofs; - struct scatterlist *out_sg; - size_t out_ofs; - - int directcall; - - u8 *src_addr; - u8 *dst_addr; - u32 dma_size; - u8 tbuf[AES_BLOCK_SIZE]; -}; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) -static void s5p_ace_print_info(void) -{ - struct s5p_ace_aes_ctx *sctx = s5p_ace_dev.ctx_bc; - - printk(KERN_INFO "flags: 0x%X\n", (u32)s5p_ace_dev.flags); - s5p_ace_dump(); - if (sctx == NULL) { - printk(KERN_INFO "sctx == NULL\n"); - } else { -#ifdef CONFIG_ACE_BC_ASYNC - printk(KERN_INFO "sctx->req: 0x%08X\n", (u32)sctx->req); -#endif - printk(KERN_INFO "sctx->total: 0x%08X\n", sctx->total); - printk(KERN_INFO "sctx->dma_size: 0x%08X\n", sctx->dma_size); - } -} -#endif - -#ifdef ACE_DEBUG_HEARTBEAT -static enum hrtimer_restart s5p_ace_heartbeat_func(struct hrtimer *timer) -{ - printk(KERN_INFO "[[ACE HEARTBEAT]] -- START ----------\n"); - - s5p_ace_print_info(); - - printk(KERN_INFO "[[ACE HEARTBEAT]] -- END ------------\n"); - - hrtimer_start(&s5p_ace_dev.heartbeat, - ns_to_ktime((u64)ACE_HEARTBEAT_MS * NSEC_PER_MSEC), - HRTIMER_MODE_REL); - - return HRTIMER_NORESTART; -} -#endif - -#ifdef ACE_DEBUG_WATCHDOG -static enum hrtimer_restart s5p_ace_watchdog_bc_func(struct hrtimer *timer) -{ - printk(KERN_ERR "[[ACE WATCHDOG BC]] ============\n"); - - s5p_ace_print_info(); - - return HRTIMER_NORESTART; -} -#endif - -static void s5p_ace_resume_device(struct s5p_ace_device *dev) -{ - if (test_and_clear_bit(FLAGS_SUSPENDED, &dev->flags)) { - clear_bit(FLAGS_BC_BUSY, &dev->flags); - clear_bit(FLAGS_HASH_BUSY, &dev->flags); - -#ifdef ACE_USE_ACP - /* Set ARUSER[12:8] and AWUSER[4:0] */ - writel(0x101, dev->sss_usercon - + (PA_SSS_USER_CON & (PAGE_SIZE - 1))); -#endif - } -} - -#if defined(CONFIG_ACE_BC) -static int s5p_ace_aes_set_cipher(struct s5p_ace_aes_ctx *sctx, - u32 alg_id, u32 key_size) -{ - u32 new_status = 0; - - /* Fixed setting */ - new_status |= ACE_AES_FIFO_ON; - - if (s5p_ace_dev.cputype == TYPE_S5PV210) - new_status |= ACE_AES_KEYCNGMODE_ON; - - new_status |= ACE_AES_SWAPKEY_ON; - new_status |= ACE_AES_SWAPCNT_ON; - new_status |= ACE_AES_SWAPIV_ON; - - if (s5p_ace_dev.cputype == TYPE_EXYNOS) { - new_status |= ACE_AES_SWAPDO_ON; - new_status |= ACE_AES_SWAPDI_ON; - new_status |= ACE_AES_COUNTERSIZE_128; - } - - switch (MI_GET_MODE(alg_id)) { - case _MODE_ECB_: - new_status |= ACE_AES_OPERMODE_ECB; - break; - case _MODE_CBC_: - new_status |= ACE_AES_OPERMODE_CBC; - break; - case _MODE_CTR_: - new_status |= ACE_AES_OPERMODE_CTR; - break; - default: - return -EINVAL; - } - - switch (key_size) { - case 128: - new_status |= ACE_AES_KEYSIZE_128; - break; - case 192: - new_status |= ACE_AES_KEYSIZE_192; - break; - case 256: - new_status |= ACE_AES_KEYSIZE_256; - break; - default: - return -EINVAL; - } - - /* Set AES context */ - sctx->sfr_ctrl = new_status; - sctx->keylen = key_size >> 3; - - return 0; -} - -/* - * enc: BC_MODE_ENC - encryption, BC_MODE_DEC - decryption - */ -static int s5p_ace_aes_set_encmode(struct s5p_ace_aes_ctx *sctx, u32 enc) -{ - u32 status = sctx->sfr_ctrl; - u32 enc_mode = ACE_AES_MODE_ENC; - - if ((status & ACE_AES_OPERMODE_MASK) != ACE_AES_OPERMODE_CTR) - enc_mode = (enc == BC_MODE_ENC ? - ACE_AES_MODE_ENC : ACE_AES_MODE_DEC); - - sctx->sfr_ctrl = (status & ~ACE_AES_MODE_MASK) | enc_mode; - - return 0; -} - -static int s5p_ace_aes_update_semikey(struct s5p_ace_aes_ctx *sctx, - u8 *in, u8 *out, u32 len) -{ - u32 *addr = (u32 *)sctx->sfr_semikey; - u32 tmp1, tmp2; - - switch (sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) { - case ACE_AES_OPERMODE_ECB: - break; - case ACE_AES_OPERMODE_CBC: - if ((sctx->sfr_ctrl & ACE_AES_MODE_MASK) == ACE_AES_MODE_ENC) - memcpy(sctx->sfr_semikey, out, AES_BLOCK_SIZE); - else - memcpy(sctx->sfr_semikey, in, AES_BLOCK_SIZE); - break; - case ACE_AES_OPERMODE_CTR: - tmp1 = be32_to_cpu(addr[3]); - tmp2 = tmp1 + (len >> 4); - addr[3] = be32_to_cpu(tmp2); - if (tmp2 < tmp1) { - tmp1 = be32_to_cpu(addr[2]) + 1; - addr[2] = be32_to_cpu(tmp1); - if (addr[2] == 0) { - tmp1 = be32_to_cpu(addr[1]) + 1; - addr[1] = be32_to_cpu(tmp1); - if (addr[1] == 0) { - tmp1 = be32_to_cpu(addr[0]) + 1; - addr[0] = be32_to_cpu(tmp1); - } - } - } - break; - default: - return -EINVAL; - } - - return 0; -} - -static int s5p_ace_aes_write_sfr(struct s5p_ace_aes_ctx *sctx) -{ - u32 *addr; - - s5p_ace_write_sfr(ACE_AES_CONTROL, sctx->sfr_ctrl); - - addr = (u32 *)sctx->sfr_key; - switch (sctx->keylen) { - case 16: - s5p_ace_write_sfr(ACE_AES_KEY5, addr[0]); - s5p_ace_write_sfr(ACE_AES_KEY6, addr[1]); - s5p_ace_write_sfr(ACE_AES_KEY7, addr[2]); - s5p_ace_write_sfr(ACE_AES_KEY8, addr[3]); - break; - case 24: - s5p_ace_write_sfr(ACE_AES_KEY3, addr[0]); - s5p_ace_write_sfr(ACE_AES_KEY4, addr[1]); - s5p_ace_write_sfr(ACE_AES_KEY5, addr[2]); - s5p_ace_write_sfr(ACE_AES_KEY6, addr[3]); - s5p_ace_write_sfr(ACE_AES_KEY7, addr[4]); - s5p_ace_write_sfr(ACE_AES_KEY8, addr[5]); - break; - case 32: - s5p_ace_write_sfr(ACE_AES_KEY1, addr[0]); - s5p_ace_write_sfr(ACE_AES_KEY2, addr[1]); - s5p_ace_write_sfr(ACE_AES_KEY3, addr[2]); - s5p_ace_write_sfr(ACE_AES_KEY4, addr[3]); - s5p_ace_write_sfr(ACE_AES_KEY5, addr[4]); - s5p_ace_write_sfr(ACE_AES_KEY6, addr[5]); - s5p_ace_write_sfr(ACE_AES_KEY7, addr[6]); - s5p_ace_write_sfr(ACE_AES_KEY8, addr[7]); - break; - default: - return -EINVAL; - } - - addr = (u32 *)sctx->sfr_semikey; - switch (sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) { - case ACE_AES_OPERMODE_ECB: - break; - case ACE_AES_OPERMODE_CBC: - s5p_ace_write_sfr(ACE_AES_IV1, addr[0]); - s5p_ace_write_sfr(ACE_AES_IV2, addr[1]); - s5p_ace_write_sfr(ACE_AES_IV3, addr[2]); - s5p_ace_write_sfr(ACE_AES_IV4, addr[3]); - break; - case ACE_AES_OPERMODE_CTR: - s5p_ace_write_sfr(ACE_AES_CNT1, addr[0]); - s5p_ace_write_sfr(ACE_AES_CNT2, addr[1]); - s5p_ace_write_sfr(ACE_AES_CNT3, addr[2]); - s5p_ace_write_sfr(ACE_AES_CNT4, addr[3]); - break; - default: - return -EINVAL; - } - - return 0; -} - -static int s5p_ace_aes_engine_start(struct s5p_ace_aes_ctx *sctx, - u8 *out, const u8 *in, u32 len, int irqen) -{ - u32 reg; - u32 first_blklen; - - if ((sctx == NULL) || (out == NULL) || (in == NULL)) { - printk(KERN_ERR "%s : NULL input.\n", __func__); - return -EINVAL; - } - - if (len & (AES_BLOCK_SIZE - 1)) { - printk(KERN_ERR "Invalid len for AES engine (%d)\n", len); - return -EINVAL; - } - - if (s5p_ace_aes_write_sfr(sctx) != 0) - return -EINVAL; - - S5P_ACE_DEBUG("AES: %s, in: 0x%08X, out: 0x%08X, len: 0x%08X\n", - __func__, (u32)in, (u32)out, len); - S5P_ACE_DEBUG("AES: %s, AES_control : 0x%08X\n", - __func__, s5p_ace_read_sfr(ACE_AES_CONTROL)); - - /* Assert code */ - reg = s5p_ace_read_sfr(ACE_AES_STATUS); - if ((reg & ACE_AES_BUSY_MASK) == ACE_AES_BUSY_ON) - return -EBUSY; - - /* Flush BRDMA and BTDMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAC, ACE_FC_BRDMACFLUSH_ON); - s5p_ace_write_sfr(ACE_FC_BTDMAC, ACE_FC_BTDMACFLUSH_ON); - - /* Select Input MUX as AES */ - reg = s5p_ace_read_sfr(ACE_FC_FIFOCTRL); - reg = (reg & ~ACE_FC_SELBC_MASK) | ACE_FC_SELBC_AES; - s5p_ace_write_sfr(ACE_FC_FIFOCTRL, reg); - - /* Stop flushing BRDMA and BTDMA */ - reg = ACE_FC_BRDMACFLUSH_OFF; - if (s5p_ace_dev.cputype == TYPE_S5PV210) - reg |= ACE_FC_BRDMACSWAP_ON; - -#ifdef ACE_USE_ACP - reg |= ACE_ARCACHE << ACE_FC_BRDMACARCACHE_OFS; -#endif - s5p_ace_write_sfr(ACE_FC_BRDMAC, reg); - reg = ACE_FC_BTDMACFLUSH_OFF; - if (s5p_ace_dev.cputype == TYPE_S5PV210) - reg |= ACE_FC_BTDMACSWAP_ON; - -#ifdef ACE_USE_ACP - reg |= ACE_AWCACHE << ACE_FC_BTDMACAWCACHE_OFS; -#endif - s5p_ace_write_sfr(ACE_FC_BTDMAC, reg); - - /* Set DMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAS, (u32)in); - s5p_ace_write_sfr(ACE_FC_BTDMAS, (u32)out); - - if (s5p_ace_dev.cputype == TYPE_S5PV210) { - /* Set the length of first block (Key Change Mode On) */ - if ((((u32)in) & (2 * AES_BLOCK_SIZE - 1)) == 0) - first_blklen = 2 * AES_BLOCK_SIZE; - else - first_blklen = AES_BLOCK_SIZE; - - if (len <= first_blklen) { -#ifdef CONFIG_ACE_BC_IRQMODE - if (irqen) - s5p_ace_write_sfr(ACE_FC_INTENSET, ACE_FC_BTDMA); -#endif - - /* Set DMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAL, len); - s5p_ace_write_sfr(ACE_FC_BTDMAL, len); - } else { - unsigned long timeout; - - /* Set DMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAL, first_blklen); - s5p_ace_write_sfr(ACE_FC_BTDMAL, first_blklen); - - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (s5p_ace_read_sfr(ACE_FC_INTPEND) & ACE_FC_BTDMA) - break; - } - if (!(s5p_ace_read_sfr(ACE_FC_INTPEND) & ACE_FC_BTDMA)) { - printk(KERN_ERR "AES : DMA time out\n"); - return -EBUSY; - } - s5p_ace_write_sfr(ACE_FC_INTPEND, ACE_FC_BTDMA | ACE_FC_BRDMA); - - reg = sctx->sfr_ctrl; - reg = (reg & ~ACE_AES_KEYCNGMODE_MASK) | ACE_AES_KEYCNGMODE_OFF; - s5p_ace_write_sfr(ACE_AES_CONTROL, reg); - -#ifdef CONFIG_ACE_BC_IRQMODE - if (irqen) - s5p_ace_write_sfr(ACE_FC_INTENSET, ACE_FC_BTDMA); -#endif - - /* Set DMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAL, len - first_blklen); - s5p_ace_write_sfr(ACE_FC_BTDMAL, len - first_blklen); - } - } else { -#ifdef CONFIG_ACE_BC_IRQMODE - if (irqen) - s5p_ace_write_sfr(ACE_FC_INTENSET, ACE_FC_BTDMA); -#endif - - /* Set DMA */ - s5p_ace_write_sfr(ACE_FC_BRDMAL, len); - s5p_ace_write_sfr(ACE_FC_BTDMAL, len); - } - - return 0; -} - -static void s5p_ace_aes_engine_wait(struct s5p_ace_aes_ctx *sctx, - u8 *out, const u8 *in, u32 len) -{ - unsigned long timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) - if (s5p_ace_read_sfr(ACE_FC_INTPEND) & ACE_FC_BTDMA) - break; - if (!(s5p_ace_read_sfr(ACE_FC_INTPEND) & ACE_FC_BTDMA)) - printk(KERN_ERR "%s : DMA time out\n", __func__); - s5p_ace_write_sfr(ACE_FC_INTPEND, ACE_FC_BTDMA | ACE_FC_BRDMA); -} - -void s5p_ace_sg_update(struct scatterlist **sg, size_t *offset, - size_t count) -{ - *offset += count; - if (*offset >= sg_dma_len(*sg)) { - *offset -= sg_dma_len(*sg); - *sg = scatterwalk_sg_next(*sg); - } -} - -int s5p_ace_sg_set_from_sg(struct scatterlist *dst, struct scatterlist *src, - u32 num) -{ - sg_init_table(dst, num); - while (num--) { - sg_set_page(dst, sg_page(src), sg_dma_len(src), src->offset); - - dst++; - src = scatterwalk_sg_next(src); - if (!src) - return -ENOMEM; - } - return 0; -} - -/* Unaligned data Handling - * - size should be a multiple of ACE_AES_MIN_BLOCK_SIZE. - */ -static int s5p_ace_aes_crypt_unaligned(struct s5p_ace_aes_ctx *sctx, - size_t size) -{ - struct blkcipher_desc desc; - struct scatterlist in_sg[2], out_sg[2]; - int ret; - - S5P_ACE_DEBUG("%s - %s (size: %d / %d)\n", __func__, - sctx->fallback_bc->base.__crt_alg->cra_driver_name, - size, sctx->total); - - desc.tfm = sctx->fallback_bc; - desc.info = sctx->sfr_semikey; - desc.flags = 0; - - s5p_ace_sg_set_from_sg(in_sg, sctx->in_sg, 2); - in_sg->length -= sctx->in_ofs; - in_sg->offset += sctx->in_ofs; - - s5p_ace_sg_set_from_sg(out_sg, sctx->out_sg, 2); - out_sg->length -= sctx->out_ofs; - out_sg->offset += sctx->out_ofs; - - if ((sctx->sfr_ctrl & ACE_AES_MODE_MASK) == ACE_AES_MODE_ENC) - ret = crypto_blkcipher_encrypt_iv( - &desc, out_sg, in_sg, size); - else - ret = crypto_blkcipher_decrypt_iv( - &desc, out_sg, in_sg, size); - - sctx->dma_size = 0; - sctx->total -= size; - if (!sctx->total) - return 0; - - s5p_ace_sg_update(&sctx->in_sg, &sctx->in_ofs, size); - s5p_ace_sg_update(&sctx->out_sg, &sctx->out_ofs, size); - - return 0; -} - -static int s5p_ace_aes_crypt_dma_start(struct s5p_ace_device *dev) -{ - struct s5p_ace_aes_ctx *sctx = dev->ctx_bc; - u8 *src, *dst; - size_t count; - int i; - int ret; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[1]); /* 1: dma start */ -#endif - - sctx->directcall = 0; - - while (1) { - count = sctx->total; - count = min(count, sg_dma_len(sctx->in_sg) - sctx->in_ofs); - count = min(count, sg_dma_len(sctx->out_sg) - sctx->out_ofs); - - S5P_ACE_DEBUG("total_start: %d (%d)\n", sctx->total, count); - S5P_ACE_DEBUG(" in(ofs: %x, len: %x), %x\n", - sctx->in_sg->offset, sg_dma_len(sctx->in_sg), - sctx->in_ofs); - S5P_ACE_DEBUG(" out(ofs: %x, len: %x), %x\n", - sctx->out_sg->offset, sg_dma_len(sctx->out_sg), - sctx->out_ofs); - - if (count > ACE_AES_MIN_BLOCK_SIZE) - break; - - count = min(sctx->total, (size_t)ACE_AES_MIN_BLOCK_SIZE); - if (count & (AES_BLOCK_SIZE - 1)) - printk(KERN_ERR "%s - Invalid count\n", __func__); - ret = s5p_ace_aes_crypt_unaligned(sctx, count); - if (!sctx->total) { -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[2]); /* 2: dma end */ -#endif -#ifdef CONFIG_ACE_BC_IRQMODE - tasklet_schedule(&dev->task_bc); - return 0; -#else - goto run; -#endif - } - } - - count &= ~(AES_BLOCK_SIZE - 1); - sctx->dma_size = count; - - src = (u8 *)page_to_phys(sg_page(sctx->in_sg)); - src += sctx->in_sg->offset + sctx->in_ofs; - if (!PageHighMem(sg_page(sctx->in_sg))) { - sctx->src_addr = (u8 *)phys_to_virt((u32)src); - } else { - sctx->src_addr = crypto_kmap(sg_page(sctx->in_sg), - crypto_kmap_type(0)); - sctx->src_addr += sctx->in_sg->offset + sctx->in_ofs; - } - - dst = (u8 *)page_to_phys(sg_page(sctx->out_sg)); - dst += sctx->out_sg->offset + sctx->out_ofs; - if (!PageHighMem(sg_page(sctx->out_sg))) { - sctx->dst_addr = (u8 *)phys_to_virt((u32)dst); - } else { - sctx->dst_addr = crypto_kmap(sg_page(sctx->out_sg), - crypto_kmap_type(1)); - sctx->dst_addr += sctx->out_sg->offset + sctx->out_ofs; - } - - S5P_ACE_DEBUG(" phys(src: %x, dst: %x)\n", (u32)src, (u32)dst); - S5P_ACE_DEBUG(" virt(src: %x, dst: %x)\n", - (u32)sctx->src_addr, (u32)sctx->dst_addr); - - if (src == dst) - memcpy(sctx->tbuf, sctx->src_addr + count - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - -#ifndef ACE_USE_ACP -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) - dmac_clean_range((void *)sctx->src_addr, - (void *)sctx->src_addr + count); - dmac_clean_range((void *)sctx->dst_addr, - (void *)sctx->dst_addr + count); -#else - dmac_map_area((void *)sctx->src_addr, count, DMA_TO_DEVICE); - outer_clean_range((unsigned long)src, (unsigned long)src + count); - dmac_map_area((void *)sctx->dst_addr, count, DMA_FROM_DEVICE); - outer_clean_range((unsigned long)dst, (unsigned long)dst + count); -#endif -#endif - - for (i = 0; i < 100; i++) { - ret = s5p_ace_aes_engine_start(sctx, dst, src, count, 1); - if (ret != -EBUSY) - break; - } - if (i == 100) { - printk(KERN_ERR "%s : DMA Start Failed\n", __func__); - return ret; - } - -run: -#ifdef CONFIG_ACE_BC_ASYNC -#ifndef CONFIG_ACE_BC_IRQMODE - if (!ret) { - if ((count <= 2048) && ((s5p_ace_dev.rc_depth_bc++) < 1)) { - sctx->directcall = 1; - s5p_ace_bc_task((unsigned long)&s5p_ace_dev); - return ret; - } - } -#endif - - if (sctx->dma_size) { - if (PageHighMem(sg_page(sctx->in_sg))) - crypto_kunmap(sctx->src_addr, crypto_kmap_type(0)); - if (PageHighMem(sg_page(sctx->out_sg))) - crypto_kunmap(sctx->dst_addr, crypto_kmap_type(1)); - } - -#ifndef CONFIG_ACE_BC_IRQMODE - if (!ret) - tasklet_schedule(&dev->task_bc); -#endif -#endif - return ret; -} - -static int s5p_ace_aes_crypt_dma_wait(struct s5p_ace_device *dev) -{ - struct s5p_ace_aes_ctx *sctx = dev->ctx_bc; - u8 *src, *dst; - u8 *src_lb_addr; - u32 lastblock; - int ret = 0; - - S5P_ACE_DEBUG("%s\n", __func__); - - src = (u8 *)page_to_phys(sg_page(sctx->in_sg)); - src += sctx->in_sg->offset + sctx->in_ofs; - dst = (u8 *)page_to_phys(sg_page(sctx->out_sg)); - dst += sctx->out_sg->offset + sctx->out_ofs; - -#ifdef CONFIG_ACE_BC_ASYNC - if (!sctx->directcall) { - if (PageHighMem(sg_page(sctx->in_sg))) { - sctx->src_addr = crypto_kmap(sg_page(sctx->in_sg), - crypto_kmap_type(0)); - sctx->src_addr += sctx->in_sg->offset + sctx->in_ofs; - } - - if (PageHighMem(sg_page(sctx->out_sg))) { - sctx->dst_addr = crypto_kmap(sg_page(sctx->out_sg), - crypto_kmap_type(1)); - sctx->dst_addr += sctx->out_sg->offset + sctx->out_ofs; - } - } -#endif - -#ifndef CONFIG_ACE_BC_IRQMODE - s5p_ace_aes_engine_wait(sctx, dst, src, sctx->dma_size); -#endif - -#ifndef ACE_USE_ACP -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) - dmac_inv_range((void *)sctx->dst_addr, - (void *)sctx->dst_addr + sctx->dma_size); -#else - dmac_unmap_area((void *)sctx->dst_addr, sctx->dma_size, - DMA_FROM_DEVICE); - outer_inv_range((unsigned long)dst, - (unsigned long)dst + sctx->dma_size); -#endif -#endif - - lastblock = sctx->dma_size - AES_BLOCK_SIZE; - if (src == dst) - src_lb_addr = sctx->tbuf; - else - src_lb_addr = sctx->src_addr + lastblock; - if (s5p_ace_aes_update_semikey(sctx, src_lb_addr, - sctx->dst_addr + lastblock, - sctx->dma_size) != 0) - return -EINVAL; - - if (PageHighMem(sg_page(sctx->in_sg))) - crypto_kunmap(sctx->src_addr, crypto_kmap_type(0)); - if (PageHighMem(sg_page(sctx->out_sg))) - crypto_kunmap(sctx->dst_addr, crypto_kmap_type(1)); - - sctx->total -= sctx->dma_size; - - S5P_ACE_DEBUG("total_end: %d\n", sctx->total); - - if (ret || !sctx->total) { - if (ret) - printk(KERN_NOTICE "err: %d\n", ret); - } else { - s5p_ace_sg_update(&sctx->in_sg, &sctx->in_ofs, - sctx->dma_size); - s5p_ace_sg_update(&sctx->out_sg, &sctx->out_ofs, - sctx->dma_size); - } - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[2]); /* 2: dma end */ -#endif - - return ret; -} - -#ifdef CONFIG_ACE_BC_ASYNC -static int s5p_ace_handle_lock_req(struct s5p_ace_device *dev, - struct s5p_ace_aes_ctx *sctx, - struct ablkcipher_request *req, u32 encmode) -{ - int ret; - - sctx->origin_tfm = req->base.tfm; - crypto_ablkcipher_set_flags(sctx->fallback_abc, 0); - ablkcipher_request_set_tfm(req, sctx->fallback_abc); - - if (encmode == BC_MODE_ENC) - ret = crypto_ablkcipher_encrypt(req); - else - ret = crypto_ablkcipher_decrypt(req); - - sctx->req = req; - dev->ctx_bc = sctx; - tasklet_schedule(&dev->task_bc); - - return ret; -} - -static int s5p_ace_aes_handle_req(struct s5p_ace_device *dev) -{ - struct crypto_async_request *async_req; - struct crypto_async_request *backlog; - struct s5p_ace_aes_ctx *sctx; - struct s5p_ace_reqctx *rctx; - struct ablkcipher_request *req; - unsigned long flags; - - if (dev->ctx_bc) - goto start; - - S5P_ACE_DEBUG("%s\n", __func__); - - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - backlog = crypto_get_backlog(&dev->queue_bc); - async_req = crypto_dequeue_request(&dev->queue_bc); - S5P_ACE_DEBUG("[[ dequeue (%u) ]]\n", dev->queue_bc.qlen); - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - if (!async_req) { - clear_bit(FLAGS_BC_BUSY, &dev->flags); - s5p_ace_clock_gating(ACE_CLOCK_OFF); - return 0; - } - - if (backlog) { - S5P_ACE_DEBUG("backlog.\n"); - backlog->complete(backlog, -EINPROGRESS); - } - - S5P_ACE_DEBUG("get new req\n"); - - req = ablkcipher_request_cast(async_req); - sctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); - -#ifdef ACE_DEBUG_WATCHDOG - hrtimer_start(&s5p_ace_dev.watchdog_bc, - ns_to_ktime((u64)ACE_WATCHDOG_MS * NSEC_PER_MSEC), - HRTIMER_MODE_REL); -#endif - rctx = ablkcipher_request_ctx(req); - - if (s5p_ace_dev.flags & BIT_MASK(FLAGS_USE_SW)) - return s5p_ace_handle_lock_req(dev, sctx, req, rctx->mode); - - /* assign new request to device */ - sctx->req = req; - sctx->total = req->nbytes; - sctx->in_sg = req->src; - sctx->in_ofs = 0; - sctx->out_sg = req->dst; - sctx->out_ofs = 0; - - if ((sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) != ACE_AES_OPERMODE_ECB) - memcpy(sctx->sfr_semikey, req->info, AES_BLOCK_SIZE); - - s5p_ace_aes_set_encmode(sctx, rctx->mode); - - dev->ctx_bc = sctx; - -start: - return s5p_ace_aes_crypt_dma_start(dev); -} - -static void s5p_ace_bc_task(unsigned long data) -{ - struct s5p_ace_device *dev = (struct s5p_ace_device *)data; - struct s5p_ace_aes_ctx *sctx = dev->ctx_bc; - int ret = 0; - - S5P_ACE_DEBUG("%s (total: %d, dma_size: %d)\n", __func__, - sctx->total, sctx->dma_size); - - /* check if it is handled by SW or HW */ - if (sctx->req->base.tfm == - crypto_ablkcipher_tfm - (crypto_ablkcipher_crt(sctx->fallback_abc)->base)) { - sctx->req->base.tfm = sctx->origin_tfm; - sctx->req->base.complete(&sctx->req->base, ret); - dev->ctx_bc = NULL; - s5p_ace_aes_handle_req(dev); - - return; - } - - if (sctx->dma_size) - ret = s5p_ace_aes_crypt_dma_wait(dev); - - if (!sctx->total) { - if ((sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) - != ACE_AES_OPERMODE_ECB) - memcpy(sctx->req->info, sctx->sfr_semikey, - AES_BLOCK_SIZE); - sctx->req->base.complete(&sctx->req->base, ret); - dev->ctx_bc = NULL; - -#ifdef ACE_DEBUG_WATCHDOG - hrtimer_cancel(&s5p_ace_dev.watchdog_bc); -#endif - } - - s5p_ace_aes_handle_req(dev); -} - -static int s5p_ace_aes_crypt(struct ablkcipher_request *req, u32 encmode) -{ - struct s5p_ace_reqctx *rctx = ablkcipher_request_ctx(req); - unsigned long flags; - int ret; - unsigned long timeout; - -#ifdef ACE_DEBUG_WATCHDOG - do_gettimeofday(×tamp[0]); /* 0: request */ -#endif - - S5P_ACE_DEBUG("%s (nbytes: 0x%x, mode: 0x%x)\n", - __func__, (u32)req->nbytes, encmode); - - rctx->mode = encmode; - - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (s5p_ace_dev.queue_bc.list.prev != &req->base.list) - break; - udelay(1); /* wait */ - } - if (s5p_ace_dev.queue_bc.list.prev == &req->base.list) { - printk(KERN_ERR "%s : Time Out.\n", __func__); - return -EAGAIN; - } - - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - ret = ablkcipher_enqueue_request(&s5p_ace_dev.queue_bc, req); - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - S5P_ACE_DEBUG("[[ enqueue (%u) ]]\n", s5p_ace_dev.queue_bc.qlen); - - s5p_ace_resume_device(&s5p_ace_dev); - if (!test_and_set_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags)) { - s5p_ace_clock_gating(ACE_CLOCK_ON); - s5p_ace_dev.rc_depth_bc = 0; - s5p_ace_aes_handle_req(&s5p_ace_dev); - } - - return ret; -} -#else -static int s5p_ace_handle_lock_req(struct s5p_ace_aes_ctx *sctx, - struct blkcipher_desc *desc, - struct scatterlist *sg_dst, - struct scatterlist *sg_src, - unsigned int size, int encmode) -{ - int ret; - - sctx->origin_tfm = desc->tfm; - desc->tfm = sctx->fallback_bc; - - if (encmode == BC_MODE_ENC) - ret = crypto_blkcipher_encrypt_iv(desc, sg_dst, sg_src, size); - else - ret = crypto_blkcipher_decrypt_iv(desc, sg_dst, sg_src, size); - - desc->tfm = sctx->origin_tfm; - - return ret; -} - -static int s5p_ace_aes_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes, int encmode) -{ - struct s5p_ace_aes_ctx *sctx = crypto_blkcipher_ctx(desc->tfm); - int ret; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[0]); /* 0: request */ -#endif - -#ifdef ACE_DEBUG_WATCHDOG - hrtimer_start(&s5p_ace_dev.watchdog_bc, - ns_to_ktime((u64)ACE_WATCHDOG_MS * NSEC_PER_MSEC), - HRTIMER_MODE_REL); -#endif - - sctx->total = nbytes; - sctx->in_sg = src; - sctx->in_ofs = 0; - sctx->out_sg = dst; - sctx->out_ofs = 0; - - if ((sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) != ACE_AES_OPERMODE_ECB) - memcpy(sctx->sfr_semikey, desc->info, AES_BLOCK_SIZE); - - s5p_ace_aes_set_encmode(sctx, encmode); - - s5p_ace_resume_device(&s5p_ace_dev); - s5p_ace_clock_gating(ACE_CLOCK_ON); - local_bh_disable(); - while (test_and_set_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags)) - udelay(1); - - if (s5p_ace_dev.flags & BIT_MASK(FLAGS_USE_SW)) { - clear_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - return s5p_ace_handle_lock_req(sctx, desc, dst, src, nbytes, - encmode); - } - - s5p_ace_dev.ctx_bc = sctx; - - do { - ret = s5p_ace_aes_crypt_dma_start(&s5p_ace_dev); - - if (sctx->dma_size) - ret = s5p_ace_aes_crypt_dma_wait(&s5p_ace_dev); - } while (sctx->total); - - s5p_ace_dev.ctx_bc = NULL; - - clear_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - s5p_ace_clock_gating(ACE_CLOCK_OFF); - - if ((sctx->sfr_ctrl & ACE_AES_OPERMODE_MASK) != ACE_AES_OPERMODE_ECB) - memcpy(desc->info, sctx->sfr_semikey, AES_BLOCK_SIZE); - -#ifdef ACE_DEBUG_WATCHDOG - hrtimer_cancel(&s5p_ace_dev.watchdog_bc); -#endif - - return ret; -} -#endif - -static int s5p_ace_aes_set_key(struct s5p_ace_aes_ctx *sctx, const u8 *key, - unsigned int key_len) -{ - memcpy(sctx->sfr_key, key, key_len); - crypto_blkcipher_setkey(sctx->fallback_bc, key, key_len); - -#ifdef CONFIG_ACE_BC_ASYNC - crypto_ablkcipher_setkey(sctx->fallback_abc, key, key_len); -#endif - - return 0; -} - -#ifdef CONFIG_ACE_BC_ASYNC -static int s5p_ace_ecb_aes_set_key(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_ablkcipher_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_ECB, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_cbc_aes_set_key(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_ablkcipher_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_CBC, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_ctr_aes_set_key(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_ablkcipher_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_CTR, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_ecb_aes_encrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_ENC); -} - -static int s5p_ace_ecb_aes_decrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_DEC); -} - -static int s5p_ace_cbc_aes_encrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_ENC); -} - -static int s5p_ace_cbc_aes_decrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_DEC); -} - -static int s5p_ace_ctr_aes_encrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_ENC); -} - -static int s5p_ace_ctr_aes_decrypt(struct ablkcipher_request *req) -{ - return s5p_ace_aes_crypt(req, BC_MODE_DEC); -} -#else -static int s5p_ace_ecb_aes_set_key(struct crypto_tfm *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_tfm_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_ECB, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_cbc_aes_set_key(struct crypto_tfm *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_tfm_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_CBC, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_ctr_aes_set_key(struct crypto_tfm *tfm, const u8 *key, - unsigned int key_len) -{ - struct s5p_ace_aes_ctx *sctx = crypto_tfm_ctx(tfm); - s5p_ace_aes_set_cipher(sctx, MI_AES_CTR, key_len * 8); - return s5p_ace_aes_set_key(sctx, key, key_len); -} - -static int s5p_ace_ecb_aes_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_ENC); -} - -static int s5p_ace_ecb_aes_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_DEC); -} - -static int s5p_ace_cbc_aes_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_ENC); -} - -static int s5p_ace_cbc_aes_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_DEC); -} - -static int s5p_ace_ctr_aes_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_ENC); -} - -static int s5p_ace_ctr_aes_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - return s5p_ace_aes_crypt(desc, dst, src, nbytes, BC_MODE_DEC); -} -#endif - -static int s5p_ace_cra_init_tfm(struct crypto_tfm *tfm) -{ - const char *name = tfm->__crt_alg->cra_name; - struct s5p_ace_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - sctx->fallback_bc = crypto_alloc_blkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - - if (IS_ERR(sctx->fallback_bc)) { - printk(KERN_ERR "Error allocating fallback algo %s\n", name); - return PTR_ERR(sctx->fallback_bc); - } -#ifdef CONFIG_ACE_BC_ASYNC - tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_ace_reqctx); - sctx->fallback_abc = crypto_alloc_ablkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - - if (IS_ERR(sctx->fallback_abc)) { - printk(KERN_ERR "Error allocating abc fallback algo %s\n", - name); - return PTR_ERR(sctx->fallback_abc); - } - -#endif - S5P_ACE_DEBUG("%s\n", __func__); - - return 0; -} - -static void s5p_ace_cra_exit_tfm(struct crypto_tfm *tfm) -{ - struct s5p_ace_aes_ctx *sctx = crypto_tfm_ctx(tfm); - - crypto_free_blkcipher(sctx->fallback_bc); - sctx->fallback_bc = NULL; - -#ifdef CONFIG_ACE_BC_ASYNC - crypto_free_ablkcipher(sctx->fallback_abc); - sctx->fallback_abc = NULL; -#endif - - S5P_ACE_DEBUG("%s\n", __func__); -} - -static struct crypto_alg algs_bc[] = { - { - .cra_name = "ecb(aes)", - .cra_driver_name = "ecb-aes-s5p-ace", - .cra_priority = 300, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER - | CRYPTO_ALG_ASYNC, -#else - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, -#endif - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s5p_ace_aes_ctx), - .cra_alignmask = 0, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_type = &crypto_ablkcipher_type, -#else - .cra_type = &crypto_blkcipher_type, -#endif - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_cra_init_tfm, - .cra_exit = s5p_ace_cra_exit_tfm, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_ablkcipher = { -#else - .cra_blkcipher = { -#endif - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = s5p_ace_ecb_aes_set_key, - .encrypt = s5p_ace_ecb_aes_encrypt, - .decrypt = s5p_ace_ecb_aes_decrypt, - } - }, - { - .cra_name = "cbc(aes)", - .cra_driver_name = "cbc-aes-s5p-ace", - .cra_priority = 300, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER - | CRYPTO_ALG_ASYNC, -#else - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, -#endif - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s5p_ace_aes_ctx), - .cra_alignmask = 0, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_type = &crypto_ablkcipher_type, -#else - .cra_type = &crypto_blkcipher_type, -#endif - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_cra_init_tfm, - .cra_exit = s5p_ace_cra_exit_tfm, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_ablkcipher = { -#else - .cra_blkcipher = { -#endif - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = s5p_ace_cbc_aes_set_key, - .encrypt = s5p_ace_cbc_aes_encrypt, - .decrypt = s5p_ace_cbc_aes_decrypt, - } - }, - { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-s5p-ace", - .cra_priority = 300, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER - | CRYPTO_ALG_ASYNC, -#else - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, -#endif - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s5p_ace_aes_ctx), - .cra_alignmask = 0, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_type = &crypto_ablkcipher_type, -#else - .cra_type = &crypto_blkcipher_type, -#endif - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_cra_init_tfm, - .cra_exit = s5p_ace_cra_exit_tfm, -#ifdef CONFIG_ACE_BC_ASYNC - .cra_ablkcipher = { -#else - .cra_blkcipher = { -#endif - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = s5p_ace_ctr_aes_set_key, - .encrypt = s5p_ace_ctr_aes_encrypt, - .decrypt = s5p_ace_ctr_aes_decrypt, - } - } -}; -#endif - -#define TYPE_HASH_SHA1 0 -#define TYPE_HASH_SHA256 1 - -#if defined(CONFIG_ACE_HASH_SHA1) || defined(CONFIG_ACE_HASH_SHA256) -struct s5p_ace_hash_ctx { - u32 type; - u32 prelen_high; - u32 prelen_low; - - u32 buflen; - u8 buffer[SHA256_BLOCK_SIZE]; - - u32 state[SHA256_DIGEST_SIZE / 4]; - - u32 sw_init; - - struct shash_desc sw_desc; - struct sha256_state dummy; -}; - -/* - * out == NULL - This is not a final message block. - * Intermediate value is stored at pCtx->digest. - * out != NULL - This is a final message block. - * Digest value will be stored at out. - */ -static int s5p_ace_sha_engine(struct s5p_ace_hash_ctx *sctx, - u8 *out, const u8* in, u32 len) -{ - u32 reg; - u32 *buffer; - u32 block_size, digest_size; - u8 *in_phys; - int transformmode = 0; - - S5P_ACE_DEBUG("Out: 0x%08X, In: 0x%08X, Len: %d\n", - (u32)out, (u32)in, len); - S5P_ACE_DEBUG("PreLen_Hi: %u, PreLen_Lo: %u\n", - sctx->prelen_high, sctx->prelen_low); - - block_size = (sctx->type == TYPE_HASH_SHA1) ? - SHA1_BLOCK_SIZE : SHA256_BLOCK_SIZE; - digest_size = (sctx->type == TYPE_HASH_SHA1) ? - SHA1_DIGEST_SIZE : SHA256_DIGEST_SIZE; - - if (out == NULL) { - if (len == 0) { - return 0; - } else if (len < digest_size) { - printk(KERN_ERR "%s: Invalid input\n", __func__); - return -EINVAL; - } - transformmode = 1; - } - - if (len == 0) { - S5P_ACE_DEBUG("%s: Workaround for empty input\n", __func__); - - memset(sctx->buffer, 0, block_size - 8); - sctx->buffer[0] = 0x80; - reg = cpu_to_be32(sctx->prelen_high); - memcpy(sctx->buffer + block_size - 8, ®, 4); - reg = cpu_to_be32(sctx->prelen_low); - memcpy(sctx->buffer + block_size - 4, ®, 4); - - in = sctx->buffer; - len = block_size; - transformmode = 1; - } - - if ((void *)in < high_memory) { - in_phys = (u8 *)virt_to_phys((void*)in); - } else { - struct page *page; - S5P_ACE_DEBUG("%s: high memory - 0x%08x\n", __func__, (u32)in); - page = vmalloc_to_page(in); - if (!page) - printk(KERN_ERR "ERROR: %s: Null page\n", __func__); - in_phys = (u8 *)page_to_phys(page); - in_phys += ((u32)in & ~PAGE_MASK); - } - - /* Flush HRDMA */ - s5p_ace_write_sfr(ACE_FC_HRDMAC, ACE_FC_HRDMACFLUSH_ON); - reg = ACE_FC_HRDMACFLUSH_OFF; - if (s5p_ace_dev.cputype == TYPE_S5PV210) - reg |= ACE_FC_HRDMACSWAP_ON; - -#ifdef ACE_USE_ACP - reg |= ACE_ARCACHE << ACE_FC_HRDMACARCACHE_OFS; -#endif - s5p_ace_write_sfr(ACE_FC_HRDMAC, reg); - - /* Set byte swap of data in */ - if (s5p_ace_dev.cputype == TYPE_EXYNOS) - s5p_ace_write_sfr(ACE_HASH_BYTESWAP, ACE_HASH_SWAPDI_ON | - ACE_HASH_SWAPDO_ON | ACE_HASH_SWAPIV_ON); - else - s5p_ace_write_sfr(ACE_HASH_BYTESWAP, - ACE_HASH_SWAPDO_ON | ACE_HASH_SWAPIV_ON); - - /* Select Hash input mux as external source */ - reg = s5p_ace_read_sfr(ACE_FC_FIFOCTRL); - reg = (reg & ~ACE_FC_SELHASH_MASK) | ACE_FC_SELHASH_EXOUT; - s5p_ace_write_sfr(ACE_FC_FIFOCTRL, reg); - - /* Set Hash as SHA1 or SHA256 and start Hash engine */ - reg = (sctx->type == TYPE_HASH_SHA1) ? - ACE_HASH_ENGSEL_SHA1HASH : ACE_HASH_ENGSEL_SHA256HASH; - reg |= ACE_HASH_STARTBIT_ON; - if ((sctx->prelen_low | sctx->prelen_high) != 0) { - reg |= ACE_HASH_USERIV_EN; - buffer = (u32 *)sctx->state; - s5p_ace_write_sfr(ACE_HASH_IV1, buffer[0]); - s5p_ace_write_sfr(ACE_HASH_IV2, buffer[1]); - s5p_ace_write_sfr(ACE_HASH_IV3, buffer[2]); - s5p_ace_write_sfr(ACE_HASH_IV4, buffer[3]); - s5p_ace_write_sfr(ACE_HASH_IV5, buffer[4]); - - if (sctx->type == TYPE_HASH_SHA256) { - s5p_ace_write_sfr(ACE_HASH_IV6, buffer[5]); - s5p_ace_write_sfr(ACE_HASH_IV7, buffer[6]); - s5p_ace_write_sfr(ACE_HASH_IV8, buffer[7]); - } - } - s5p_ace_write_sfr(ACE_HASH_CONTROL, reg); - - /* Enable FIFO mode */ - s5p_ace_write_sfr(ACE_HASH_FIFO_MODE, ACE_HASH_FIFO_ON); - - /* Clean data cache */ -#ifndef ACE_USE_ACP -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) - dmac_clean_range((void *)in, (void *)in + len); -#else - dmac_map_area((void *)in, len, DMA_TO_DEVICE); - outer_clean_range((unsigned long)in_phys, (unsigned long)in_phys + len); -#endif -#endif - - if (transformmode) { - /* Set message length */ - s5p_ace_write_sfr(ACE_HASH_MSGSIZE_LOW, 0); - s5p_ace_write_sfr(ACE_HASH_MSGSIZE_HIGH, 0x80000000); - - /* Set pre-message length */ - s5p_ace_write_sfr(ACE_HASH_PRELEN_LOW, 0); - s5p_ace_write_sfr(ACE_HASH_PRELEN_HIGH, 0); - } else { - /* Set message length */ - s5p_ace_write_sfr(ACE_HASH_MSGSIZE_LOW, len); - s5p_ace_write_sfr(ACE_HASH_MSGSIZE_HIGH, 0); - - /* Set pre-message length */ - s5p_ace_write_sfr(ACE_HASH_PRELEN_LOW, sctx->prelen_low); - s5p_ace_write_sfr(ACE_HASH_PRELEN_HIGH, sctx->prelen_high); - } - - /* Set HRDMA */ - s5p_ace_write_sfr(ACE_FC_HRDMAS, (u32)in_phys); - s5p_ace_write_sfr(ACE_FC_HRDMAL, len); - - while (!(s5p_ace_read_sfr(ACE_FC_INTPEND) & ACE_FC_HRDMA)) - ; /* wait */ - s5p_ace_write_sfr(ACE_FC_INTPEND, ACE_FC_HRDMA); - - /*while ((s5p_ace_read_sfr(ACE_HASH_STATUS) & ACE_HASH_BUFRDY_MASK) - == ACE_HASH_BUFRDY_OFF); */ - - if (transformmode) { - /* Set Pause bit */ - s5p_ace_write_sfr(ACE_HASH_CONTROL2, ACE_HASH_PAUSE_ON); - - while ((s5p_ace_read_sfr(ACE_HASH_STATUS) - & ACE_HASH_PARTIALDONE_MASK) - == ACE_HASH_PARTIALDONE_OFF) - ; /* wait */ - s5p_ace_write_sfr(ACE_HASH_STATUS, ACE_HASH_PARTIALDONE_ON); - - if (out == NULL) { - /* Update chaining variables */ - buffer = (u32 *)sctx->state; - - /* Update pre-message length */ - /* Note that the unit of pre-message length is a BIT! */ - sctx->prelen_low += (len << 3); - if (sctx->prelen_low < len) - sctx->prelen_high++; - sctx->prelen_high += (len >> 29); - } else { - /* Read hash result */ - buffer = (u32 *)out; - } - } else { - while ((s5p_ace_read_sfr(ACE_HASH_STATUS) - & ACE_HASH_MSGDONE_MASK) - == ACE_HASH_MSGDONE_OFF) - ; /* wait */ - s5p_ace_write_sfr(ACE_HASH_STATUS, ACE_HASH_MSGDONE_ON); - - /* Read hash result */ - buffer = (u32 *)out; - } - buffer[0] = s5p_ace_read_sfr(ACE_HASH_RESULT1); - buffer[1] = s5p_ace_read_sfr(ACE_HASH_RESULT2); - buffer[2] = s5p_ace_read_sfr(ACE_HASH_RESULT3); - buffer[3] = s5p_ace_read_sfr(ACE_HASH_RESULT4); - buffer[4] = s5p_ace_read_sfr(ACE_HASH_RESULT5); - - if (sctx->type == TYPE_HASH_SHA256) { - buffer[5] = s5p_ace_read_sfr(ACE_HASH_RESULT6); - buffer[6] = s5p_ace_read_sfr(ACE_HASH_RESULT7); - buffer[7] = s5p_ace_read_sfr(ACE_HASH_RESULT8); - } - - return 0; -} - -#ifdef CONFIG_ACE_HASH_ASYNC -static int s5p_ace_sha1_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct s5p_ace_hash_ctx *sctx = crypto_ahash_ctx(tfm); - - sctx->prelen_high = sctx->prelen_low = 0; - sctx->buflen = 0; - - /* To Do */ - - return 0; -} - -static int s5p_ace_sha1_update(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct s5p_ace_hash_ctx *sctx = crypto_ahash_ctx(tfm); - - /* To Do */ - - return 0; -} - -static int s5p_ace_sha1_final(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct s5p_ace_hash_ctx *sctx = crypto_ahash_ctx(tfm); - - /* To Do */ - - return 0; -} - -static int s5p_ace_sha1_finup(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct s5p_ace_hash_ctx *sctx = crypto_ahash_ctx(tfm); - - /* To Do */ - - return 0; -} - -static int s5p_ace_sha1_digest(struct ahash_request *req) -{ - s5p_ace_sha1_init(req); - s5p_ace_sha1_update(req); - s5p_ace_sha1_final(req); - - return 0; -} -#else -static void sha1_export_ctx_to_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - struct sha1_state *sw_ctx = shash_desc_ctx(&sctx->sw_desc); - int i; - - if (sctx->prelen_low == 0 && sctx->prelen_high == 0) - crypto_shash_alg(&sw_tfm[sctx->type]) - ->init(&sctx->sw_desc); - else { - for (i = 0; i < SHA1_DIGEST_SIZE/4; i++) - sw_ctx->state[i] = be32_to_cpu(sctx->state[i]); - } - - sw_ctx->count = (((u64)sctx->prelen_high << 29) | - (sctx->prelen_low >> 3)) + sctx->buflen; - - if (sctx->buflen) - memcpy(sw_ctx->buffer, sctx->buffer, sctx->buflen); -} - -static void sha256_export_ctx_to_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - struct sha256_state *sw_ctx = shash_desc_ctx(&sctx->sw_desc); - int i; - - if (sctx->prelen_low == 0 && sctx->prelen_high == 0) - crypto_shash_alg(&sw_tfm[sctx->type]) - ->init(&sctx->sw_desc); - else { - for (i = 0; i < SHA256_DIGEST_SIZE/4; i++) - sw_ctx->state[i] = be32_to_cpu(sctx->state[i]); - } - - sw_ctx->count = (((u64)sctx->prelen_high << 29) | - (sctx->prelen_low >> 3)) + sctx->buflen; - - if (sctx->buflen) - memcpy(sw_ctx->buf, sctx->buffer, sctx->buflen); -} - -static void sha1_import_ctx_from_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - struct sha1_state *sw_ctx = shash_desc_ctx(&sctx->sw_desc); - int i; - - for (i = 0; i < SHA1_DIGEST_SIZE/4; i++) - sctx->state[i] = cpu_to_be32(sw_ctx->state[i]); - - memcpy(sctx->buffer, sw_ctx->buffer, sw_ctx->count & - (SHA1_BLOCK_SIZE - 1)); - sctx->buflen = sw_ctx->count & (SHA1_BLOCK_SIZE - 1); - - sctx->prelen_low = (sw_ctx->count - sctx->buflen) << 3; - sctx->prelen_high = (sw_ctx->count - sctx->buflen) >> 29; -} - -static void sha256_import_ctx_from_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - struct sha256_state *sw_ctx = shash_desc_ctx(&sctx->sw_desc); - int i; - - for (i = 0; i < SHA256_DIGEST_SIZE/4; i++) - sctx->state[i] = cpu_to_be32(sw_ctx->state[i]); - - memcpy(sctx->buffer, sw_ctx->buf, sw_ctx->count & - (SHA256_BLOCK_SIZE - 1)); - sctx->buflen = sw_ctx->count & (SHA256_BLOCK_SIZE - 1); - - sctx->prelen_low = (sw_ctx->count - sctx->buflen) << 3; - sctx->prelen_high = (sw_ctx->count - sctx->buflen) >> 29; -} - -static void hash_export_ctx_to_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - if (!sctx->sw_init) { - sctx->sw_init = 1; - if (sctx->prelen_low == 0 && sctx->prelen_high == 0 && - sctx->buflen == 0) { - crypto_shash_alg(&sw_tfm[sctx->type]) - ->init(&sctx->sw_desc); - return; - } - } - - if (sctx->type == TYPE_HASH_SHA1) - sha1_export_ctx_to_sw(desc); - else - sha256_export_ctx_to_sw(desc); -} - -static void hash_import_ctx_from_sw(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - if (sctx->type == TYPE_HASH_SHA1) - sha1_import_ctx_from_sw(desc); - else - sha256_import_ctx_from_sw(desc); - -} - -static int sha_sw_update(struct shash_desc *desc, const u8 *data, unsigned - int len) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - hash_export_ctx_to_sw(desc); - crypto_shash_alg(&sw_tfm[sctx->type])->update(&sctx->sw_desc, data, - len); - hash_import_ctx_from_sw(desc); - - return 0; -} - -static int sha_sw_final(struct shash_desc *desc, u8 *out) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - hash_export_ctx_to_sw(desc); - crypto_shash_alg(&sw_tfm[sctx->type])->final(&sctx->sw_desc, out); - hash_import_ctx_from_sw(desc); - - return 0; -} - -static int sha_sw_finup(struct shash_desc *desc, const u8 *data, unsigned int - len, u8 *out) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - hash_export_ctx_to_sw(desc); - crypto_shash_alg(&sw_tfm[sctx->type])->update(&sctx->sw_desc, data, - len); - crypto_shash_alg(&sw_tfm[sctx->type])->final(&sctx->sw_desc, out); - hash_import_ctx_from_sw(desc); - - return 0; -} - -#if defined(CONFIG_ACE_HASH_SHA1) -static int s5p_ace_sha1_init(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - sctx->prelen_high = sctx->prelen_low = 0; - sctx->buflen = 0; - sctx->type = TYPE_HASH_SHA1; - sctx->sw_init = 0; - - return 0; -} -#endif - -#if defined(CONFIG_ACE_HASH_SHA256) -static int s5p_ace_sha256_init(struct shash_desc *desc) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - sctx->prelen_high = sctx->prelen_low = 0; - sctx->buflen = 0; - sctx->type = TYPE_HASH_SHA256; - sctx->sw_init = 0; - - return 0; -} -#endif - -static int s5p_ace_sha_update(struct shash_desc *desc, - const u8 *data, unsigned int len) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - const u8 *src; - int ret = 0; - u32 partlen, tmplen, block_size; - - S5P_ACE_DEBUG("%s (buflen: 0x%x, len: 0x%x)\n", - __func__, sctx->buflen, len); - - s5p_ace_resume_device(&s5p_ace_dev); - local_bh_disable(); - while (test_and_set_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags)) - udelay(1); - - if (s5p_ace_dev.flags & BIT_MASK(FLAGS_USE_SW)) { - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - return sha_sw_update(desc, data, len); - } - - partlen = sctx->buflen; - src = data; - - block_size = (sctx->type == TYPE_HASH_SHA1) ? - SHA1_BLOCK_SIZE : SHA256_BLOCK_SIZE; - s5p_ace_clock_gating(ACE_CLOCK_ON); - - if (partlen != 0) { - if (partlen + len < block_size) { - memcpy(sctx->buffer + partlen, src, len); - sctx->buflen += len; - goto out; - } else { - tmplen = block_size - partlen; - memcpy(sctx->buffer + partlen, src, tmplen); - - ret = s5p_ace_sha_engine(sctx, NULL, sctx->buffer, - block_size); - if (ret) - goto out; - - len -= tmplen; - src += tmplen; - } - } - - partlen = len & (block_size - 1); - len -= partlen; - if (len > 0) { - ret = s5p_ace_sha_engine(sctx, NULL, src, len); - if (ret) - goto out; - } - - memcpy(sctx->buffer, src + len, partlen); - sctx->buflen = partlen; - -out: - s5p_ace_clock_gating(ACE_CLOCK_OFF); - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - - return ret; -} - -static int s5p_ace_sha_final(struct shash_desc *desc, u8 *out) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - - S5P_ACE_DEBUG("%s (buflen: 0x%x)\n", __func__, sctx->buflen); - - s5p_ace_resume_device(&s5p_ace_dev); - local_bh_disable(); - while (test_and_set_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags)) - udelay(1); - - if (s5p_ace_dev.flags & BIT_MASK(FLAGS_USE_SW)) { - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - return sha_sw_final(desc, out); - } - - s5p_ace_clock_gating(ACE_CLOCK_ON); - s5p_ace_sha_engine(sctx, out, sctx->buffer, sctx->buflen); - s5p_ace_clock_gating(ACE_CLOCK_OFF); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - - return 0; -} - -static int s5p_ace_sha_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - const u8 *src; - int ret = 0; - u32 block_size; - - S5P_ACE_DEBUG("%s (buflen: 0x%x, len: 0x%x)\n", - __func__, sctx->buflen, len); - - s5p_ace_resume_device(&s5p_ace_dev); - local_bh_disable(); - while (test_and_set_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags)) - udelay(1); - - if (s5p_ace_dev.flags & BIT_MASK(FLAGS_USE_SW)) { - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - return sha_sw_finup(desc, data, len, out); - } - - src = data; - block_size = (sctx->type == TYPE_HASH_SHA1) ? - SHA1_BLOCK_SIZE : SHA256_BLOCK_SIZE; - - s5p_ace_clock_gating(ACE_CLOCK_ON); - - if (sctx->buflen != 0) { - if (sctx->buflen + len <= block_size) { - memcpy(sctx->buffer + sctx->buflen, src, len); - - len += sctx->buflen; - src = sctx->buffer; - } else { - u32 copylen = block_size - sctx->buflen; - memcpy(sctx->buffer + sctx->buflen, src, copylen); - - ret = s5p_ace_sha_engine(sctx, NULL, sctx->buffer, - block_size); - if (ret) - goto out; - - len -= copylen; - src += copylen; - } - } - - ret = s5p_ace_sha_engine(sctx, out, src, len); - -out: - s5p_ace_clock_gating(ACE_CLOCK_OFF); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - local_bh_enable(); - - return ret; -} - -#if defined(CONFIG_ACE_HASH_SHA1) -static int s5p_ace_sha1_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - int ret; - - ret = s5p_ace_sha1_init(desc); - if (ret) - return ret; - - return s5p_ace_sha_finup(desc, data, len, out); -} -#endif - -#if defined(CONFIG_ACE_HASH_SHA256) -static int s5p_ace_sha256_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - int ret; - - ret = s5p_ace_sha256_init(desc); - if (ret) - return ret; - - return s5p_ace_sha_finup(desc, data, len, out); -} -#endif - -static int s5p_ace_hash_export(struct shash_desc *desc, void *out) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int s5p_ace_hash_import(struct shash_desc *desc, const void *in) -{ - struct s5p_ace_hash_ctx *sctx = shash_desc_ctx(desc); - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} -#endif - -static int s5p_ace_hash_cra_init(struct crypto_tfm *tfm) -{ -#ifdef CONFIG_ACE_HASH_ASYNC -#endif - - S5P_ACE_DEBUG("%s\n", __func__); - - return 0; -} - -static void s5p_ace_hash_cra_exit(struct crypto_tfm *tfm) -{ -#ifdef CONFIG_ACE_HASH_ASYNC -#endif - - S5P_ACE_DEBUG("%s\n", __func__); -} - -#ifdef CONFIG_ACE_HASH_ASYNC -static struct ahash_alg algs_hash[] = { -#if defined(CONFIG_ACE_HASH_SHA1) - { - .init = s5p_ace_sha1_init, - .update = s5p_ace_sha_update, - .final = s5p_ace_sha_final, - .finup = s5p_ace_sha_finup, - .digest = s5p_ace_sha1_digest, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-s5p-ace", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_TYPE_AHASH - | CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s5p_ace_hash_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_hash_cra_init, - .cra_exit = s5p_ace_hash_cra_exit, - } - } -#endif -}; -#else -static struct shash_alg algs_hash[] = { -#if defined(CONFIG_ACE_HASH_SHA1) - { - .digestsize = SHA1_DIGEST_SIZE, - .init = s5p_ace_sha1_init, - .update = s5p_ace_sha_update, - .final = s5p_ace_sha_final, - .finup = s5p_ace_sha_finup, - .digest = s5p_ace_sha1_digest, - .export = s5p_ace_hash_export, - .import = s5p_ace_hash_import, - .descsize = sizeof(struct s5p_ace_hash_ctx), - .statesize = sizeof(struct s5p_ace_hash_ctx), - .base = { - .cra_name = "sha1", - .cra_driver_name = "sha1-s5p-ace", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_hash_cra_init, - .cra_exit = s5p_ace_hash_cra_exit, - } - }, -#endif -#if defined(CONFIG_ACE_HASH_SHA256) - { - .digestsize = SHA256_DIGEST_SIZE, - .init = s5p_ace_sha256_init, - .update = s5p_ace_sha_update, - .final = s5p_ace_sha_final, - .finup = s5p_ace_sha_finup, - .digest = s5p_ace_sha256_digest, - .export = s5p_ace_hash_export, - .import = s5p_ace_hash_import, - .descsize = sizeof(struct s5p_ace_hash_ctx), - .statesize = sizeof(struct s5p_ace_hash_ctx), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-s5p-ace", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - .cra_init = s5p_ace_hash_cra_init, - .cra_exit = s5p_ace_hash_cra_exit, - } - } -#endif -}; -#endif /* CONFIG_ACE_HASH_ASYNC */ -#endif /* CONFIG_ACE_HASH_SHA1 or CONFIG_ACE_HASH_SHA256 */ - -#if defined(CONFIG_ACE_BC_IRQMODE) || defined(CONFIG_ACE_HASH_IRQMODE) -static irqreturn_t s5p_ace_interrupt(int irq, void *data) -{ - struct s5p_ace_device *dev = data; - - s5p_ace_write_sfr(ACE_FC_INTPEND, - ACE_FC_BRDMA | ACE_FC_BTDMA | ACE_FC_HRDMA); - -#ifdef CONFIG_ACE_BC_IRQMODE - s5p_ace_write_sfr(ACE_FC_INTENCLR, ACE_FC_BRDMA | ACE_FC_BTDMA); - - tasklet_schedule(&dev->task_bc); -#endif - -#ifdef CONFIG_ACE_HASH_IRQMODE - s5p_ace_write_sfr(ACE_FC_INTENCLR, ACE_FC_HRDMA); -#endif - - return IRQ_HANDLED; -} -#endif - -int ace_s5p_get_sync_lock(void) -{ - unsigned long timeout; - int get_lock_bc = 0, get_lock_hash = 0; - unsigned long flags; - - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (!test_and_set_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags)) { - get_lock_bc = 1; - break; - } - udelay(1); - } - - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (!test_and_set_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags)) { - get_lock_hash = 1; - break; - } - udelay(1); - } - - /* set lock flag */ - if (get_lock_bc && get_lock_hash) { - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - count_use_sw++; - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - set_bit(FLAGS_USE_SW, &s5p_ace_dev.flags); - } - - if (get_lock_bc) { -#ifdef CONFIG_ACE_BC_ASYNC - if (s5p_ace_dev.queue_bc.qlen > 0) { - s5p_ace_clock_gating(ACE_CLOCK_ON); - s5p_ace_dev.rc_depth_bc = 0; - s5p_ace_aes_handle_req(&s5p_ace_dev); - } else { - clear_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags); - } -#else - clear_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags); -#endif - } - - if (get_lock_hash) - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - - if (!(get_lock_bc && get_lock_hash)) - return -EBUSY; - - s5p_ace_clock_gating(ACE_CLOCK_ON); - - return 0; -} - -int ace_s5p_release_sync_lock(void) -{ - unsigned long flags; - - spin_lock_irqsave(&s5p_ace_dev.lock, flags); - count_use_sw--; - spin_unlock_irqrestore(&s5p_ace_dev.lock, flags); - - /* clear lock flag */ - if (!count_use_sw) - clear_bit(FLAGS_USE_SW, &s5p_ace_dev.flags); - - s5p_ace_clock_gating(ACE_CLOCK_OFF); - - return 0; -} - -static int __devinit s5p_ace_probe(struct platform_device *pdev) -{ - struct resource *res; - struct s5p_ace_device *s5p_adt = &s5p_ace_dev; - int i, j, k, m; - int ret; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp_base); - for (i = 0; i < 5; i++) - do_gettimeofday(×tamp[i]); -#endif - - memset(s5p_adt, 0, sizeof(*s5p_adt)); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) { - dev_err(&pdev->dev, "failed to get platform resource\n"); - return -ENOENT; - } - - s5p_adt->ace_base = ioremap(res->start, resource_size(res)); - if (s5p_adt->ace_base == NULL) { - dev_err(&pdev->dev, "failed to remap register block\n"); - ret = -ENOMEM; - goto err_mem1; - } - - s5p_adt->clock = clk_get(&pdev->dev, "secss"); - if (IS_ERR(s5p_adt->clock)) { - dev_err(&pdev->dev, "failed to find clock source\n"); - ret = -EBUSY; - goto err_clk; - } - s5p_ace_init_clock_gating(); - s5p_adt->cputype = platform_get_device_id(pdev)->driver_data; - -#if defined(CONFIG_ACE_BC_IRQMODE) || defined(CONFIG_ACE_HASH_IRQMODE) - s5p_adt->irq = platform_get_irq(pdev, 0); - if (s5p_adt->irq < 0) { - dev_err(&pdev->dev, "Failed to get irq#\n"); - s5p_adt->irq = 0; - ret = -ENODEV; - goto err_irq; - } - ret = request_irq(s5p_adt->irq, s5p_ace_interrupt, 0, - S5P_ACE_DRIVER_NAME, (void *)s5p_adt); - if (ret) { - dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", - s5p_adt->irq, ret); - s5p_adt->irq = 0; - ret = -ENODEV; - goto err_irq; - } -#endif - -#ifdef ACE_USE_ACP - s5p_adt->sss_usercon = ioremap(PA_SSS_USER_CON & PAGE_MASK, SZ_4K); - if (s5p_adt->sss_usercon == NULL) { - dev_err(&pdev->dev, "failed to remap register SSS_USER_CON\n"); - ret = -EBUSY; - goto err_mem2; - } - - /* Set ARUSER[12:8] and AWUSER[4:0] */ - writel(0x101, s5p_adt->sss_usercon - + (PA_SSS_USER_CON & (PAGE_SIZE - 1))); -#endif - - spin_lock_init(&s5p_adt->lock); - s5p_adt->flags = 0; - hrtimer_init(&s5p_adt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - s5p_adt->timer.function = s5p_ace_timer_func; - INIT_WORK(&s5p_adt->work, s5p_ace_deferred_clock_disable); -#ifdef ACE_DEBUG_HEARTBEAT - hrtimer_init(&s5p_adt->heartbeat, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - s5p_adt->heartbeat.function = s5p_ace_heartbeat_func; - hrtimer_start(&s5p_ace_dev.heartbeat, - ns_to_ktime((u64)ACE_HEARTBEAT_MS * NSEC_PER_MSEC), - HRTIMER_MODE_REL); -#endif -#ifdef ACE_DEBUG_WATCHDOG - hrtimer_init(&s5p_adt->watchdog_bc, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - s5p_adt->watchdog_bc.function = s5p_ace_watchdog_bc_func; -#endif - -#ifdef CONFIG_ACE_BC_ASYNC - crypto_init_queue(&s5p_adt->queue_bc, 1); - tasklet_init(&s5p_adt->task_bc, s5p_ace_bc_task, - (unsigned long)s5p_adt); -#endif - -#ifdef CONFIG_ACE_HASH_ASYNC - crypto_init_queue(&s5p_adt->queue_hash, 1); - tasklet_init(&s5p_adt->task_hash, s5p_ace_hash_task, - (unsigned long)s5p_adt); -#endif - -#if defined(CONFIG_ACE_BC) - for (i = 0; i < ARRAY_SIZE(algs_bc); i++) { - INIT_LIST_HEAD(&algs_bc[i].cra_list); - algs_bc[i].cra_flags |= CRYPTO_ALG_NEED_FALLBACK; - ret = crypto_register_alg(&algs_bc[i]); - if (ret) - goto err_reg_bc; - printk(KERN_INFO "ACE: %s\n", algs_bc[i].cra_driver_name); - } -#endif - -#if defined(CONFIG_ACE_HASH_SHA1) || defined(CONFIG_ACE_HASH_SHA256) - fallback_hash = (struct crypto_hash **) - kmalloc(sizeof(struct crypto_hash *) * - ARRAY_SIZE(algs_hash), GFP_KERNEL); - sw_tfm = (struct crypto_shash *) kmalloc(sizeof(struct crypto_shash) - * ARRAY_SIZE(algs_hash), - GFP_KERNEL); - - for (m = 0; m < ARRAY_SIZE(algs_hash); m++) { - fallback_hash[m] = - crypto_alloc_hash(algs_hash[m].base.cra_name, 0, - CRYPTO_ALG_ASYNC); - - if (IS_ERR(fallback_hash[m])) { - printk(KERN_ERR "failed to load transform for %s: %ld\n", - algs_hash[m].base.cra_name, - PTR_ERR(fallback_hash[m])); - goto err_fallback_hash; - } - - sw_tfm[m].base.__crt_alg = fallback_hash[m]->base.__crt_alg; - } - - for (j = 0; j < ARRAY_SIZE(algs_hash); j++) { -#ifdef CONFIG_ACE_HASH_ASYNC - ret = crypto_register_ahash(&algs_hash[j]); -#else - ret = crypto_register_shash(&algs_hash[j]); -#endif - if (ret) - goto err_reg_hash; -#ifdef CONFIG_ACE_HASH_ASYNC - printk(KERN_INFO "ACE: %s\n", - algs_hash[j].halg.base.cra_driver_name); -#else - printk(KERN_INFO "ACE: %s\n", - algs_hash[j].base.cra_driver_name); -#endif - } -#endif - - secmem_ftn.lock = &ace_s5p_get_sync_lock; - secmem_ftn.release = &ace_s5p_release_sync_lock; - secmem_crypto_register(&secmem_ftn); - - count_use_sw = 0; - - printk(KERN_NOTICE "ACE driver is initialized\n"); - - return 0; - -#if defined(CONFIG_ACE_HASH_SHA1) || defined(CONFIG_ACE_HASH_SHA256) -err_reg_hash: - for (k = 0; k < j; k++) -#ifdef CONFIG_ACE_HASH_ASYNC - crypto_unregister_ahash(&algs_hash[k]); -#else - crypto_unregister_shash(&algs_hash[k]); -#endif -err_fallback_hash: - kfree(sw_tfm); - for (k = 0; k < m; k++) - crypto_free_hash(fallback_hash[k]); - kfree(fallback_hash); -#endif -#if defined(CONFIG_ACE_BC) -err_reg_bc: - for (k = 0; k < i; k++) - crypto_unregister_alg(&algs_bc[k]); -#ifdef CONFIG_ACE_BC_ASYNC - tasklet_kill(&s5p_adt->task_bc); -#endif -#endif -#ifdef CONFIG_ACE_HASH_ASYNC - tasklet_kill(&s5p_adt->task_hash); -#endif -#ifdef ACE_USE_ACP - iounmap(s5p_adt->sss_usercon); -err_mem2: -#endif -#if defined(CONFIG_ACE_BC_IRQMODE) || defined(CONFIG_ACE_HASH_IRQMODE) -err_irq: - free_irq(s5p_adt->irq, (void *)s5p_adt); - s5p_adt->irq = 0; -#endif -err_clk: - iounmap(s5p_adt->ace_base); - s5p_adt->ace_base = NULL; -err_mem1: - - printk(KERN_ERR "ACE driver initialization failed.\n"); - - return ret; -} - -static int s5p_ace_remove(struct platform_device *dev) -{ - struct s5p_ace_device *s5p_adt = &s5p_ace_dev; - int i; - -#ifdef ACE_DEBUG_HEARTBEAT - hrtimer_cancel(&s5p_adt->heartbeat); -#endif - -#if defined(CONFIG_ACE_BC_IRQMODE) || defined(CONFIG_ACE_HASH_IRQMODE) - if (s5p_adt->irq) { - free_irq(s5p_adt->irq, (void *)s5p_adt); - s5p_adt->irq = 0; - } -#endif - - if (s5p_adt->clock) { - clk_put(s5p_adt->clock); - s5p_adt->clock = NULL; - } - - if (s5p_adt->ace_base) { - iounmap(s5p_adt->ace_base); - s5p_adt->ace_base = NULL; - } - -#ifdef ACE_USE_ACP - if (s5p_adt->sss_usercon) { - iounmap(s5p_adt->sss_usercon); - s5p_adt->sss_usercon = NULL; - } -#endif - - secmem_crypto_deregister(); - -#if defined(CONFIG_ACE_HASH_SHA1) || defined(CONFIG_ACE_HASH_SHA256) - kfree(sw_tfm); - for (i = 0; i < ARRAY_SIZE(algs_hash); i++) - crypto_free_hash(fallback_hash[i]); - - kfree(fallback_hash); - - for (i = 0; i < ARRAY_SIZE(algs_hash); i++) -#ifdef CONFIG_ACE_HASH_ASYNC - crypto_unregister_ahash(&algs_hash[i]); -#else - crypto_unregister_shash(&algs_hash[i]); -#endif -#endif - -#if defined(CONFIG_ACE_BC) - for (i = 0; i < ARRAY_SIZE(algs_bc); i++) - crypto_unregister_alg(&algs_bc[i]); - -#ifdef CONFIG_ACE_BC_ASYNC - tasklet_kill(&s5p_adt->task_bc); -#endif -#endif -#ifdef CONFIG_ACE_HASH_ASYNC - tasklet_kill(&s5p_adt->task_hash); -#endif - - flush_work(&s5p_ace_dev.work); - - printk(KERN_INFO "ACE driver is removed\n"); - - return 0; -} - -static int s5p_ace_suspend(struct platform_device *dev, pm_message_t state) -{ - unsigned long timeout; - int get_lock_bc = 0, get_lock_hash = 0; - -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[3]); /* 3: suspend */ -#endif - - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (!test_and_set_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags)) { - get_lock_bc = 1; - break; - } - udelay(1); - } - timeout = jiffies + msecs_to_jiffies(10); - while (time_before(jiffies, timeout)) { - if (!test_and_set_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags)) { - get_lock_hash = 1; - break; - } - udelay(1); - } - - if (get_lock_bc && get_lock_hash) { - set_bit(FLAGS_SUSPENDED, &s5p_ace_dev.flags); - return 0; - } - - printk(KERN_ERR "ACE: suspend: time out.\n"); - - if (get_lock_bc) - clear_bit(FLAGS_BC_BUSY, &s5p_ace_dev.flags); - if (get_lock_hash) - clear_bit(FLAGS_HASH_BUSY, &s5p_ace_dev.flags); - - return -EBUSY; -} - -static int s5p_ace_resume(struct platform_device *dev) -{ -#if defined(ACE_DEBUG_HEARTBEAT) || defined(ACE_DEBUG_WATCHDOG) - do_gettimeofday(×tamp[4]); /* 4: resume */ -#endif - - s5p_ace_resume_device(&s5p_ace_dev); - - return 0; -} - -static struct platform_device_id s5p_ace_driver_ids[] = { - { - .name = "s5pv210-ace", - .driver_data = TYPE_S5PV210, - }, { - .name = "exynos-ace", - .driver_data = TYPE_EXYNOS, - }, - {} -}; -MODULE_DEVICE_TABLE(platform, s5p_ace_driver_ids); - -static struct platform_driver s5p_ace_driver = { - .probe = s5p_ace_probe, - .remove = s5p_ace_remove, - .suspend = s5p_ace_suspend, - .resume = s5p_ace_resume, - .id_table = s5p_ace_driver_ids, - .driver = { - .name = S5P_ACE_DRIVER_NAME, - .owner = THIS_MODULE, - }, -}; - -static int __init s5p_ace_init(void) -{ - printk(KERN_INFO "S5P ACE Driver, (c) 2010 Samsung Electronics\n"); - - return platform_driver_register(&s5p_ace_driver); -} - -static void __exit s5p_ace_exit(void) -{ - platform_driver_unregister(&s5p_ace_driver); -} - -module_init(s5p_ace_init); -module_exit(s5p_ace_exit); - -MODULE_DESCRIPTION("S5P ACE(Advanced Crypto Engine) support"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Dong Jin PARK"); - |