diff options
Diffstat (limited to 'arch/arm/kernel/entry-armv.S')
| -rw-r--r-- | arch/arm/kernel/entry-armv.S | 678 |
1 files changed, 363 insertions, 315 deletions
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index ece0996..2cd0076 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -24,59 +24,26 @@ #include <asm/unwind.h> #include <asm/unistd.h> #include <asm/tls.h> -#include <asm/system.h> #include "entry-header.S" #include <asm/entry-macro-multi.S> /* - * Interrupt handling. + * Interrupt handling. Preserves r7, r8, r9 */ .macro irq_handler #ifdef CONFIG_MULTI_IRQ_HANDLER - ldr r1, =handle_arch_irq + ldr r5, =handle_arch_irq mov r0, sp - ldr r1, [r1] + ldr r5, [r5] adr lr, BSYM(9997f) - teq r1, #0 - movne pc, r1 + teq r5, #0 + movne pc, r5 #endif arch_irq_handler_default 9997: .endm - .macro pabt_helper - @ PABORT handler takes pt_regs in r2, fault address in r4 and psr in r5 -#ifdef MULTI_PABORT - ldr ip, .LCprocfns - mov lr, pc - ldr pc, [ip, #PROCESSOR_PABT_FUNC] -#else - bl CPU_PABORT_HANDLER -#endif - .endm - - .macro dabt_helper - - @ - @ Call the processor-specific abort handler: - @ - @ r2 - pt_regs - @ r4 - aborted context pc - @ r5 - aborted context psr - @ - @ The abort handler must return the aborted address in r0, and - @ the fault status register in r1. r9 must be preserved. - @ -#ifdef MULTI_DABORT - ldr ip, .LCprocfns - mov lr, pc - ldr pc, [ip, #PROCESSOR_DABT_FUNC] -#else - bl CPU_DABORT_HANDLER -#endif - .endm - #ifdef CONFIG_KPROBES .section .kprobes.text,"ax",%progbits #else @@ -159,74 +126,106 @@ ENDPROC(__und_invalid) SPFIX( subeq sp, sp, #4 ) stmia sp, {r1 - r12} - ldmia r0, {r3 - r5} - add r7, sp, #S_SP - 4 @ here for interlock avoidance - mov r6, #-1 @ "" "" "" "" - add r2, sp, #(S_FRAME_SIZE + \stack_hole - 4) - SPFIX( addeq r2, r2, #4 ) - str r3, [sp, #-4]! @ save the "real" r0 copied + ldmia r0, {r1 - r3} + add r5, sp, #S_SP - 4 @ here for interlock avoidance + mov r4, #-1 @ "" "" "" "" + add r0, sp, #(S_FRAME_SIZE + \stack_hole - 4) + SPFIX( addeq r0, r0, #4 ) + str r1, [sp, #-4]! @ save the "real" r0 copied @ from the exception stack - mov r3, lr + mov r1, lr @ @ We are now ready to fill in the remaining blanks on the stack: @ - @ r2 - sp_svc - @ r3 - lr_svc - @ r4 - lr_<exception>, already fixed up for correct return/restart - @ r5 - spsr_<exception> - @ r6 - orig_r0 (see pt_regs definition in ptrace.h) + @ r0 - sp_svc + @ r1 - lr_svc + @ r2 - lr_<exception>, already fixed up for correct return/restart + @ r3 - spsr_<exception> + @ r4 - orig_r0 (see pt_regs definition in ptrace.h) @ - stmia r7, {r2 - r6} - -#ifdef CONFIG_TRACE_IRQFLAGS - bl trace_hardirqs_off -#endif + stmia r5, {r0 - r4} .endm .align 5 __dabt_svc: svc_entry + + @ + @ get ready to re-enable interrupts if appropriate + @ + mrs r9, cpsr + tst r3, #PSR_I_BIT + biceq r9, r9, #PSR_I_BIT + + @ + @ Call the processor-specific abort handler: + @ + @ r2 - aborted context pc + @ r3 - aborted context cpsr + @ + @ The abort handler must return the aborted address in r0, and + @ the fault status register in r1. r9 must be preserved. + @ +#ifdef MULTI_DABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_DABT_FUNC] +#else + bl CPU_DABORT_HANDLER +#endif + + @ + @ set desired IRQ state, then call main handler + @ + debug_entry r1 + msr cpsr_c, r9 mov r2, sp - dabt_helper + bl do_DataAbort @ @ IRQs off again before pulling preserved data off the stack @ disable_irq_notrace -#ifdef CONFIG_TRACE_IRQFLAGS - tst r5, #PSR_I_BIT - bleq trace_hardirqs_on - tst r5, #PSR_I_BIT - blne trace_hardirqs_off -#endif - svc_exit r5 @ return from exception + @ + @ restore SPSR and restart the instruction + @ + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__dabt_svc) .align 5 __irq_svc: svc_entry - irq_handler +#ifdef CONFIG_TRACE_IRQFLAGS + bl trace_hardirqs_off +#endif #ifdef CONFIG_PREEMPT get_thread_info tsk ldr r8, [tsk, #TI_PREEMPT] @ get preempt count + add r7, r8, #1 @ increment it + str r7, [tsk, #TI_PREEMPT] +#endif + + irq_handler +#ifdef CONFIG_PREEMPT + str r8, [tsk, #TI_PREEMPT] @ restore preempt count ldr r0, [tsk, #TI_FLAGS] @ get flags teq r8, #0 @ if preempt count != 0 movne r0, #0 @ force flags to 0 tst r0, #_TIF_NEED_RESCHED blne svc_preempt #endif - + ldr r4, [sp, #S_PSR] @ irqs are already disabled #ifdef CONFIG_TRACE_IRQFLAGS - @ The parent context IRQs must have been enabled to get here in - @ the first place, so there's no point checking the PSR I bit. - bl trace_hardirqs_on + tst r4, #PSR_I_BIT + bleq trace_hardirqs_on #endif - svc_exit r5 @ return from exception + svc_exit r4 @ return from exception UNWIND(.fnend ) ENDPROC(__irq_svc) @@ -242,19 +241,6 @@ svc_preempt: b 1b #endif -__und_fault: - @ Correct the PC such that it is pointing at the instruction - @ which caused the fault. If the faulting instruction was ARM - @ the PC will be pointing at the next instruction, and have to - @ subtract 4. Otherwise, it is Thumb, and the PC will be - @ pointing at the second half of the Thumb instruction. We - @ have to subtract 2. - ldr r2, [r0, #S_PC] - sub r2, r2, r1 - str r2, [r0, #S_PC] - b do_undefinstr -ENDPROC(__und_fault) - .align 5 __und_svc: #ifdef CONFIG_KPROBES @@ -265,6 +251,7 @@ __und_svc: #else svc_entry #endif + @ @ call emulation code, which returns using r9 if it has emulated @ the instruction, or the more conventional lr if we are to treat @@ -272,65 +259,68 @@ __und_svc: @ @ r0 - instruction @ -#ifndef CONFIG_THUMB2_KERNEL - ldr r0, [r4, #-4] +#ifndef CONFIG_THUMB2_KERNEL + ldr r0, [r2, #-4] #else - mov r1, #2 - ldrh r0, [r4, #-2] @ Thumb instruction at LR - 2 - cmp r0, #0xe800 @ 32-bit instruction if xx >= 0 - blo __und_svc_fault - ldrh r9, [r4] @ bottom 16 bits - add r4, r4, #2 - str r4, [sp, #S_PC] - orr r0, r9, r0, lsl #16 + ldrh r0, [r2, #-2] @ Thumb instruction at LR - 2 + and r9, r0, #0xf800 + cmp r9, #0xe800 @ 32-bit instruction if xx >= 0 + ldrhhs r9, [r2] @ bottom 16 bits + orrhs r0, r9, r0, lsl #16 #endif - adr r9, BSYM(__und_svc_finish) - mov r2, r4 + adr r9, BSYM(1f) bl call_fpe - mov r1, #4 @ PC correction to apply -__und_svc_fault: mov r0, sp @ struct pt_regs *regs - bl __und_fault + bl do_undefinstr @ @ IRQs off again before pulling preserved data off the stack @ -__und_svc_finish: - disable_irq_notrace +1: disable_irq_notrace @ @ restore SPSR and restart the instruction @ - ldr r5, [sp, #S_PSR] @ Get SVC cpsr -#ifdef CONFIG_TRACE_IRQFLAGS - tst r5, #PSR_I_BIT - bleq trace_hardirqs_on - tst r5, #PSR_I_BIT - blne trace_hardirqs_off -#endif - svc_exit r5 @ return from exception + ldr r2, [sp, #S_PSR] @ Get SVC cpsr + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__und_svc) .align 5 __pabt_svc: svc_entry + + @ + @ re-enable interrupts if appropriate + @ + mrs r9, cpsr + tst r3, #PSR_I_BIT + biceq r9, r9, #PSR_I_BIT + + mov r0, r2 @ pass address of aborted instruction. +#ifdef MULTI_PABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_PABT_FUNC] +#else + bl CPU_PABORT_HANDLER +#endif + debug_entry r1 + msr cpsr_c, r9 @ Maybe enable interrupts mov r2, sp @ regs - pabt_helper + bl do_PrefetchAbort @ call abort handler @ @ IRQs off again before pulling preserved data off the stack @ disable_irq_notrace -#ifdef CONFIG_TRACE_IRQFLAGS - tst r5, #PSR_I_BIT - bleq trace_hardirqs_on - tst r5, #PSR_I_BIT - blne trace_hardirqs_off -#endif - svc_exit r5 @ return from exception + @ + @ restore SPSR and restart the instruction + @ + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception UNWIND(.fnend ) ENDPROC(__pabt_svc) @@ -361,23 +351,23 @@ ENDPROC(__pabt_svc) ARM( stmib sp, {r1 - r12} ) THUMB( stmia sp, {r0 - r12} ) - ldmia r0, {r3 - r5} + ldmia r0, {r1 - r3} add r0, sp, #S_PC @ here for interlock avoidance - mov r6, #-1 @ "" "" "" "" + mov r4, #-1 @ "" "" "" "" - str r3, [sp] @ save the "real" r0 copied + str r1, [sp] @ save the "real" r0 copied @ from the exception stack @ @ We are now ready to fill in the remaining blanks on the stack: @ - @ r4 - lr_<exception>, already fixed up for correct return/restart - @ r5 - spsr_<exception> - @ r6 - orig_r0 (see pt_regs definition in ptrace.h) + @ r2 - lr_<exception>, already fixed up for correct return/restart + @ r3 - spsr_<exception> + @ r4 - orig_r0 (see pt_regs definition in ptrace.h) @ @ Also, separately save sp_usr and lr_usr @ - stmia r0, {r4 - r6} + stmia r0, {r2 - r4} ARM( stmdb r0, {sp, lr}^ ) THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) @@ -390,14 +380,10 @@ ENDPROC(__pabt_svc) @ Clear FP to mark the first stack frame @ zero_fp - -#ifdef CONFIG_IRQSOFF_TRACER - bl trace_hardirqs_off -#endif .endm .macro kuser_cmpxchg_check -#if !defined(CONFIG_CPU_32v6K) && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) +#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) #ifndef CONFIG_MMU #warning "NPTL on non MMU needs fixing" #else @@ -405,8 +391,8 @@ ENDPROC(__pabt_svc) @ if it was interrupted in a critical region. Here we @ perform a quick test inline since it should be false @ 99.9999% of the time. The rest is done out of line. - cmp r4, #TASK_SIZE - blhs kuser_cmpxchg64_fixup + cmp r2, #TASK_SIZE + blhs kuser_cmpxchg_fixup #endif #endif .endm @@ -415,9 +401,32 @@ ENDPROC(__pabt_svc) __dabt_usr: usr_entry kuser_cmpxchg_check + + @ + @ Call the processor-specific abort handler: + @ + @ r2 - aborted context pc + @ r3 - aborted context cpsr + @ + @ The abort handler must return the aborted address in r0, and + @ the fault status register in r1. + @ +#ifdef MULTI_DABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_DABT_FUNC] +#else + bl CPU_DABORT_HANDLER +#endif + + @ + @ IRQs on, then call the main handler + @ + debug_entry r1 + enable_irq mov r2, sp - dabt_helper - b ret_from_exception + adr lr, BSYM(ret_from_exception) + b do_DataAbort UNWIND(.fnend ) ENDPROC(__dabt_usr) @@ -425,8 +434,28 @@ ENDPROC(__dabt_usr) __irq_usr: usr_entry kuser_cmpxchg_check - irq_handler + +#ifdef CONFIG_IRQSOFF_TRACER + bl trace_hardirqs_off +#endif + get_thread_info tsk +#ifdef CONFIG_PREEMPT + ldr r8, [tsk, #TI_PREEMPT] @ get preempt count + add r7, r8, #1 @ increment it + str r7, [tsk, #TI_PREEMPT] +#endif + + irq_handler +#ifdef CONFIG_PREEMPT + ldr r0, [tsk, #TI_PREEMPT] + str r8, [tsk, #TI_PREEMPT] + teq r0, r7 + ARM( strne r0, [r0, -r0] ) + THUMB( movne r0, #0 ) + THUMB( strne r0, [r0] ) +#endif + mov why, #0 b ret_to_user_from_irq UNWIND(.fnend ) @@ -438,93 +467,55 @@ ENDPROC(__irq_usr) __und_usr: usr_entry - mov r2, r4 - mov r3, r5 - - @ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the - @ faulting instruction depending on Thumb mode. - @ r3 = regs->ARM_cpsr @ - @ The emulation code returns using r9 if it has emulated the - @ instruction, or the more conventional lr if we are to treat - @ this as a real undefined instruction + @ fall through to the emulation code, which returns using r9 if + @ it has emulated the instruction, or the more conventional lr + @ if we are to treat this as a real undefined instruction + @ + @ r0 - instruction @ adr r9, BSYM(ret_from_exception) - + adr lr, BSYM(__und_usr_unknown) tst r3, #PSR_T_BIT @ Thumb mode? - bne __und_usr_thumb - sub r4, r2, #4 @ ARM instr at LR - 4 -1: ldrt r0, [r4] + itet eq @ explicit IT needed for the 1f label + subeq r4, r2, #4 @ ARM instr at LR - 4 + subne r4, r2, #2 @ Thumb instr at LR - 2 +1: ldreqt r0, [r4] #ifdef CONFIG_CPU_ENDIAN_BE8 - rev r0, r0 @ little endian instruction + reveq r0, r0 @ little endian instruction #endif - @ r0 = 32-bit ARM instruction which caused the exception - @ r2 = PC value for the following instruction (:= regs->ARM_pc) - @ r4 = PC value for the faulting instruction - @ lr = 32-bit undefined instruction function - adr lr, BSYM(__und_usr_fault_32) - b call_fpe - -__und_usr_thumb: + beq call_fpe @ Thumb instruction - sub r4, r2, #2 @ First half of thumb instr at LR - 2 -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 -/* - * Thumb-2 instruction handling. Note that because pre-v6 and >= v6 platforms - * can never be supported in a single kernel, this code is not applicable at - * all when __LINUX_ARM_ARCH__ < 6. This allows simplifying assumptions to be - * made about .arch directives. - */ -#if __LINUX_ARM_ARCH__ < 7 -/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */ -#define NEED_CPU_ARCHITECTURE - ldr r5, .LCcpu_architecture - ldr r5, [r5] - cmp r5, #CPU_ARCH_ARMv7 - blo __und_usr_fault_16 @ 16bit undefined instruction -/* - * The following code won't get run unless the running CPU really is v7, so - * coding round the lack of ldrht on older arches is pointless. Temporarily - * override the assembler target arch with the minimum required instead: - */ - .arch armv6t2 -#endif -2: ldrht r5, [r4] - cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16 @ 16bit undefined instruction -3: ldrht r0, [r2] +#if __LINUX_ARM_ARCH__ >= 7 +2: + ARM( ldrht r5, [r4], #2 ) + THUMB( ldrht r5, [r4] ) + THUMB( add r4, r4, #2 ) + and r0, r5, #0xf800 @ mask bits 111x x... .... .... + cmp r0, #0xe800 @ 32bit instruction if xx != 0 + blo __und_usr_unknown +3: ldrht r0, [r4] add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 - str r2, [sp, #S_PC] @ it's a 2x16bit instr, update - orr r0, r0, r5, lsl #16 - adr lr, BSYM(__und_usr_fault_32) - @ r0 = the two 16-bit Thumb instructions which caused the exception - @ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc) - @ r4 = PC value for the first 16-bit Thumb instruction - @ lr = 32bit undefined instruction function - -#if __LINUX_ARM_ARCH__ < 7 -/* If the target arch was overridden, change it back: */ -#ifdef CONFIG_CPU_32v6K - .arch armv6k + orr r0, r0, r5, lsl #16 #else - .arch armv6 -#endif -#endif /* __LINUX_ARM_ARCH__ < 7 */ -#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */ - b __und_usr_fault_16 + b __und_usr_unknown #endif - UNWIND(.fnend) + UNWIND(.fnend ) ENDPROC(__und_usr) + @ + @ fallthrough to call_fpe + @ + /* - * The out of line fixup for the ldrt instructions above. + * The out of line fixup for the ldrt above. */ .pushsection .fixup, "ax" 4: mov pc, r9 .popsection .pushsection __ex_table,"a" .long 1b, 4b -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 +#if __LINUX_ARM_ARCH__ >= 7 .long 2b, 4b .long 3b, 4b #endif @@ -548,12 +539,11 @@ ENDPROC(__und_usr) * NEON handler code. * * Emulators may wish to make use of the following registers: - * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) - * r2 = PC value to resume execution after successful emulation + * r0 = instruction opcode. + * r2 = PC+4 * r9 = normal "successful" return address - * r10 = this threads thread_info structure + * r10 = this threads thread_info structure. * lr = unrecognised instruction return address - * IRQs disabled, FIQs enabled. */ @ @ Fall-through from Thumb-2 __und_usr @@ -634,12 +624,6 @@ call_fpe: movw_pc lr @ CP#14 (Debug) movw_pc lr @ CP#15 (Control) -#ifdef NEED_CPU_ARCHITECTURE - .align 2 -.LCcpu_architecture: - .word __cpu_architecture -#endif - #ifdef CONFIG_NEON .align 6 @@ -688,23 +672,29 @@ ENTRY(no_fp) mov pc, lr ENDPROC(no_fp) -__und_usr_fault_32: - mov r1, #4 - b 1f -__und_usr_fault_16: - mov r1, #2 -1: enable_irq +__und_usr_unknown: + enable_irq mov r0, sp adr lr, BSYM(ret_from_exception) - b __und_fault -ENDPROC(__und_usr_fault_32) -ENDPROC(__und_usr_fault_16) + b do_undefinstr +ENDPROC(__und_usr_unknown) .align 5 __pabt_usr: usr_entry + + mov r0, r2 @ pass address of aborted instruction. +#ifdef MULTI_PABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_PABT_FUNC] +#else + bl CPU_PABORT_HANDLER +#endif + debug_entry r1 + enable_irq @ Enable interrupts mov r2, sp @ regs - pabt_helper + bl do_PrefetchAbort @ call abort handler UNWIND(.fnend ) /* fall through */ /* @@ -768,12 +758,31 @@ ENDPROC(__switch_to) /* * User helpers. * + * These are segment of kernel provided user code reachable from user space + * at a fixed address in kernel memory. This is used to provide user space + * with some operations which require kernel help because of unimplemented + * native feature and/or instructions in many ARM CPUs. The idea is for + * this code to be executed directly in user mode for best efficiency but + * which is too intimate with the kernel counter part to be left to user + * libraries. In fact this code might even differ from one CPU to another + * depending on the available instruction set and restrictions like on + * SMP systems. In other words, the kernel reserves the right to change + * this code as needed without warning. Only the entry points and their + * results are guaranteed to be stable. + * * Each segment is 32-byte aligned and will be moved to the top of the high * vector page. New segments (if ever needed) must be added in front of * existing ones. This mechanism should be used only for things that are * really small and justified, and not be abused freely. * - * See Documentation/arm/kernel_user_helpers.txt for formal definitions. + * User space is expected to implement those things inline when optimizing + * for a processor that has the necessary native support, but only if such + * resulting binaries are already to be incompatible with earlier ARM + * processors due to the use of unsupported instructions other than what + * is provided here. In other words don't make binaries unable to run on + * earlier processors just for the sake of not using these kernel helpers + * if your compiled code is not going to use the new instructions for other + * purpose. */ THUMB( .arm ) @@ -790,104 +799,97 @@ ENDPROC(__switch_to) __kuser_helper_start: /* - * Due to the length of some sequences, __kuser_cmpxchg64 spans 2 regular - * kuser "slots", therefore 0xffff0f80 is not used as a valid entry point. + * Reference prototype: + * + * void __kernel_memory_barrier(void) + * + * Input: + * + * lr = return address + * + * Output: + * + * none + * + * Clobbered: + * + * none + * + * Definition and user space usage example: + * + * typedef void (__kernel_dmb_t)(void); + * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0) + * + * Apply any needed memory barrier to preserve consistency with data modified + * manually and __kuser_cmpxchg usage. + * + * This could be used as follows: + * + * #define __kernel_dmb() \ + * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \ + * : : : "r0", "lr","cc" ) */ -__kuser_cmpxchg64: @ 0xffff0f60 - -#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) - - /* - * Poor you. No fast solution possible... - * The kernel itself must perform the operation. - * A special ghost syscall is used for that (see traps.c). - */ - stmfd sp!, {r7, lr} - ldr r7, 1f @ it's 20 bits - swi __ARM_NR_cmpxchg64 - ldmfd sp!, {r7, pc} -1: .word __ARM_NR_cmpxchg64 - -#elif defined(CONFIG_CPU_32v6K) - - stmfd sp!, {r4, r5, r6, r7} - ldrd r4, r5, [r0] @ load old val - ldrd r6, r7, [r1] @ load new val - smp_dmb arm -1: ldrexd r0, r1, [r2] @ load current val - eors r3, r0, r4 @ compare with oldval (1) - eoreqs r3, r1, r5 @ compare with oldval (2) - strexdeq r3, r6, r7, [r2] @ store newval if eq - teqeq r3, #1 @ success? - beq 1b @ if no then retry - smp_dmb arm - rsbs r0, r3, #0 @ set returned val and C flag - ldmfd sp!, {r4, r5, r6, r7} - bx lr - -#elif !defined(CONFIG_SMP) - -#ifdef CONFIG_MMU - - /* - * The only thing that can break atomicity in this cmpxchg64 - * implementation is either an IRQ or a data abort exception - * causing another process/thread to be scheduled in the middle of - * the critical sequence. The same strategy as for cmpxchg is used. - */ - stmfd sp!, {r4, r5, r6, lr} - ldmia r0, {r4, r5} @ load old val - ldmia r1, {r6, lr} @ load new val -1: ldmia r2, {r0, r1} @ load current val - eors r3, r0, r4 @ compare with oldval (1) - eoreqs r3, r1, r5 @ compare with oldval (2) -2: stmeqia r2, {r6, lr} @ store newval if eq - rsbs r0, r3, #0 @ set return val and C flag - ldmfd sp!, {r4, r5, r6, pc} - - .text -kuser_cmpxchg64_fixup: - @ Called from kuser_cmpxchg_fixup. - @ r4 = address of interrupted insn (must be preserved). - @ sp = saved regs. r7 and r8 are clobbered. - @ 1b = first critical insn, 2b = last critical insn. - @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b. - mov r7, #0xffff0fff - sub r7, r7, #(0xffff0fff - (0xffff0f60 + (1b - __kuser_cmpxchg64))) - subs r8, r4, r7 - rsbcss r8, r8, #(2b - 1b) - strcs r7, [sp, #S_PC] -#if __LINUX_ARM_ARCH__ < 6 - bcc kuser_cmpxchg32_fixup -#endif - mov pc, lr - .previous - -#else -#warning "NPTL on non MMU needs fixing" - mov r0, #-1 - adds r0, r0, #0 - usr_ret lr -#endif - -#else -#error "incoherent kernel configuration" -#endif - - /* pad to next slot */ - .rept (16 - (. - __kuser_cmpxchg64)/4) - .word 0 - .endr - - .align 5 - __kuser_memory_barrier: @ 0xffff0fa0 smp_dmb arm usr_ret lr .align 5 +/* + * Reference prototype: + * + * int __kernel_cmpxchg(int oldval, int newval, int *ptr) + * + * Input: + * + * r0 = oldval + * r1 = newval + * r2 = ptr + * lr = return address + * + * Output: + * + * r0 = returned value (zero or non-zero) + * C flag = set if r0 == 0, clear if r0 != 0 + * + * Clobbered: + * + * r3, ip, flags + * + * Definition and user space usage example: + * + * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr); + * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0) + * + * Atomically store newval in *ptr if *ptr is equal to oldval for user space. + * Return zero if *ptr was changed or non-zero if no exchange happened. + * The C flag is also set if *ptr was changed to allow for assembly + * optimization in the calling code. + * + * Notes: + * + * - This routine already includes memory barriers as needed. + * + * For example, a user space atomic_add implementation could look like this: + * + * #define atomic_add(ptr, val) \ + * ({ register unsigned int *__ptr asm("r2") = (ptr); \ + * register unsigned int __result asm("r1"); \ + * asm volatile ( \ + * "1: @ atomic_add\n\t" \ + * "ldr r0, [r2]\n\t" \ + * "mov r3, #0xffff0fff\n\t" \ + * "add lr, pc, #4\n\t" \ + * "add r1, r0, %2\n\t" \ + * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \ + * "bcc 1b" \ + * : "=&r" (__result) \ + * : "r" (__ptr), "rIL" (val) \ + * : "r0","r3","ip","lr","cc","memory" ); \ + * __result; }) + */ + __kuser_cmpxchg: @ 0xffff0fc0 #if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) @@ -923,15 +925,15 @@ __kuser_cmpxchg: @ 0xffff0fc0 usr_ret lr .text -kuser_cmpxchg32_fixup: +kuser_cmpxchg_fixup: @ Called from kuser_cmpxchg_check macro. - @ r4 = address of interrupted insn (must be preserved). + @ r2 = address of interrupted insn (must be preserved). @ sp = saved regs. r7 and r8 are clobbered. @ 1b = first critical insn, 2b = last critical insn. - @ If r4 >= 1b and r4 <= 2b then saved pc_usr is set to 1b. + @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. mov r7, #0xffff0fff sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) - subs r8, r4, r7 + subs r8, r2, r7 rsbcss r8, r8, #(2b - 1b) strcs r7, [sp, #S_PC] mov pc, lr @@ -961,6 +963,39 @@ kuser_cmpxchg32_fixup: .align 5 +/* + * Reference prototype: + * + * int __kernel_get_tls(void) + * + * Input: + * + * lr = return address + * + * Output: + * + * r0 = TLS value + * + * Clobbered: + * + * none + * + * Definition and user space usage example: + * + * typedef int (__kernel_get_tls_t)(void); + * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0) + * + * Get the TLS value as previously set via the __ARM_NR_set_tls syscall. + * + * This could be used as follows: + * + * #define __kernel_get_tls() \ + * ({ register unsigned int __val asm("r0"); \ + * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \ + * : "=r" (__val) : : "lr","cc" ); \ + * __val; }) + */ + __kuser_get_tls: @ 0xffff0fe0 ldr r0, [pc, #(16 - 8)] @ read TLS, set in kuser_get_tls_init usr_ret lr @@ -969,6 +1004,19 @@ __kuser_get_tls: @ 0xffff0fe0 .word 0 @ 0xffff0ff0 software TLS value, then .endr @ pad up to __kuser_helper_version +/* + * Reference declaration: + * + * extern unsigned int __kernel_helper_version; + * + * Definition and user space usage example: + * + * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc) + * + * User space may read this to determine the curent number of helpers + * available. + */ + __kuser_helper_version: @ 0xffff0ffc .word ((__kuser_helper_end - __kuser_helper_start) >> 5) |