From b01c8b54a1a271c0fc4243845927fe1d250767a3 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Wed, 29 Jun 2011 00:18:26 +0000 Subject: powerpc, KVM: Rework KVM checks in first-level interrupt handlers Instead of branching out-of-line with the DO_KVM macro to check if we are in a KVM guest at the time of an interrupt, this moves the KVM check inline in the first-level interrupt handlers. This speeds up the non-KVM case and makes sure that none of the interrupt handlers are missing the check. Because the first-level interrupt handlers are now larger, some things had to be move out of line in exceptions-64s.S. This all necessitated some minor changes to the interrupt entry code in KVM. This also streamlines the book3s_32 KVM test. Signed-off-by: Paul Mackerras Signed-off-by: Alexander Graf --- arch/powerpc/include/asm/exception-64s.h | 121 +++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 40 deletions(-) (limited to 'arch/powerpc/include/asm/exception-64s.h') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index f5dfe34..b6a3a44 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -61,19 +61,22 @@ #define EXC_HV H #define EXC_STD -#define EXCEPTION_PROLOG_1(area) \ +#define __EXCEPTION_PROLOG_1(area, extra, vec) \ GET_PACA(r13); \ std r9,area+EX_R9(r13); /* save r9 - r12 */ \ std r10,area+EX_R10(r13); \ - std r11,area+EX_R11(r13); \ - std r12,area+EX_R12(r13); \ BEGIN_FTR_SECTION_NESTED(66); \ mfspr r10,SPRN_CFAR; \ std r10,area+EX_CFAR(r13); \ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ - GET_SCRATCH0(r9); \ - std r9,area+EX_R13(r13); \ - mfcr r9 + mfcr r9; \ + extra(vec); \ + std r11,area+EX_R11(r13); \ + std r12,area+EX_R12(r13); \ + GET_SCRATCH0(r10); \ + std r10,area+EX_R13(r13) +#define EXCEPTION_PROLOG_1(area, extra, vec) \ + __EXCEPTION_PROLOG_1(area, extra, vec) #define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ ld r12,PACAKBASE(r13); /* get high part of &label */ \ @@ -85,13 +88,54 @@ mtspr SPRN_##h##SRR1,r10; \ h##rfid; \ b . /* prevent speculative execution */ -#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ +#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ __EXCEPTION_PROLOG_PSERIES_1(label, h) -#define EXCEPTION_PROLOG_PSERIES(area, label, h) \ - EXCEPTION_PROLOG_1(area); \ +#define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec) \ + EXCEPTION_PROLOG_1(area, extra, vec); \ EXCEPTION_PROLOG_PSERIES_1(label, h); +#define __KVMTEST(n) \ + lbz r10,PACA_KVM_SVCPU+SVCPU_IN_GUEST(r13); \ + cmpwi r10,0; \ + bne do_kvm_##n + +#define __KVM_HANDLER(area, h, n) \ +do_kvm_##n: \ + ld r10,area+EX_R10(r13); \ + stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \ + ld r9,area+EX_R9(r13); \ + std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \ + li r12,n; \ + b kvmppc_interrupt + +#define __KVM_HANDLER_SKIP(area, h, n) \ +do_kvm_##n: \ + cmpwi r10,KVM_GUEST_MODE_SKIP; \ + ld r10,area+EX_R10(r13); \ + beq 89f; \ + stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \ + ld r9,area+EX_R9(r13); \ + std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \ + li r12,n; \ + b kvmppc_interrupt; \ +89: mtocrf 0x80,r9; \ + ld r9,area+EX_R9(r13); \ + b kvmppc_skip_##h##interrupt + +#ifdef CONFIG_KVM_BOOK3S_64_HANDLER +#define KVMTEST(n) __KVMTEST(n) +#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n) +#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n) + +#else +#define KVMTEST(n) +#define KVM_HANDLER(area, h, n) +#define KVM_HANDLER_SKIP(area, h, n) +#endif + +#define NOTEST(n) + /* * The common exception prolog is used for all except a few exceptions * such as a segment miss on a kernel address. We have to be prepared @@ -164,57 +208,54 @@ .globl label##_pSeries; \ label##_pSeries: \ HMT_MEDIUM; \ - DO_KVM vec; \ SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_STD) + EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ + EXC_STD, KVMTEST, vec) #define STD_EXCEPTION_HV(loc, vec, label) \ . = loc; \ .globl label##_hv; \ label##_hv: \ HMT_MEDIUM; \ - DO_KVM vec; \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_HV) + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ + EXC_HV, KVMTEST, vec) -#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ - HMT_MEDIUM; \ - DO_KVM vec; \ - SET_SCRATCH0(r13); /* save r13 */ \ - GET_PACA(r13); \ - std r9,PACA_EXGEN+EX_R9(r13); /* save r9, r10 */ \ - std r10,PACA_EXGEN+EX_R10(r13); \ +#define __SOFTEN_TEST(h) \ lbz r10,PACASOFTIRQEN(r13); \ - mfcr r9; \ cmpwi r10,0; \ - beq masked_##h##interrupt; \ - GET_SCRATCH0(r10); \ - std r10,PACA_EXGEN+EX_R13(r13); \ - std r11,PACA_EXGEN+EX_R11(r13); \ - std r12,PACA_EXGEN+EX_R12(r13); \ - ld r12,PACAKBASE(r13); /* get high part of &label */ \ - ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label##_common) \ - mtspr SPRN_##h##SRR0,r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - mtspr SPRN_##h##SRR1,r10; \ - h##rfid; \ - b . /* prevent speculative execution */ -#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ - __MASKABLE_EXCEPTION_PSERIES(vec, label, h) + beq masked_##h##interrupt +#define _SOFTEN_TEST(h) __SOFTEN_TEST(h) + +#define SOFTEN_TEST(vec) \ + KVMTEST(vec); \ + _SOFTEN_TEST(EXC_STD) + +#define SOFTEN_TEST_HV(vec) \ + KVMTEST(vec); \ + _SOFTEN_TEST(EXC_HV) + +#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ + HMT_MEDIUM; \ + SET_SCRATCH0(r13); /* save r13 */ \ + __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \ + EXCEPTION_PROLOG_PSERIES_1(label##_common, h); +#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ + __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) #define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \ . = loc; \ .globl label##_pSeries; \ label##_pSeries: \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_STD) + _MASKABLE_EXCEPTION_PSERIES(vec, label, \ + EXC_STD, SOFTEN_TEST) #define MASKABLE_EXCEPTION_HV(loc, vec, label) \ . = loc; \ .globl label##_hv; \ label##_hv: \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_HV) + _MASKABLE_EXCEPTION_PSERIES(vec, label, \ + EXC_HV, SOFTEN_TEST_HV) #ifdef CONFIG_PPC_ISERIES #define DISABLE_INTS \ -- cgit v1.1 From 3c42bf8a717cb636e0ed2ed77194669e2ac3ed56 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Wed, 29 Jun 2011 00:20:58 +0000 Subject: KVM: PPC: Split host-state fields out of kvmppc_book3s_shadow_vcpu There are several fields in struct kvmppc_book3s_shadow_vcpu that temporarily store bits of host state while a guest is running, rather than anything relating to the particular guest or vcpu. This splits them out into a new kvmppc_host_state structure and modifies the definitions in asm-offsets.c to suit. On 32-bit, we have a kvmppc_host_state structure inside the kvmppc_book3s_shadow_vcpu since the assembly code needs to be able to get to them both with one pointer. On 64-bit they are separate fields in the PACA. This means that on 64-bit we don't need to copy the kvmppc_host_state in and out on vcpu load/unload, and in future will mean that the book3s_hv code doesn't need a shadow_vcpu struct in the PACA at all. That does mean that we have to be careful not to rely on any values persisting in the hstate field of the paca across any point where we could block or get preempted. Signed-off-by: Paul Mackerras Signed-off-by: Alexander Graf --- arch/powerpc/include/asm/exception-64s.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'arch/powerpc/include/asm/exception-64s.h') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index b6a3a44..296c9b6 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -96,16 +96,16 @@ EXCEPTION_PROLOG_PSERIES_1(label, h); #define __KVMTEST(n) \ - lbz r10,PACA_KVM_SVCPU+SVCPU_IN_GUEST(r13); \ + lbz r10,HSTATE_IN_GUEST(r13); \ cmpwi r10,0; \ bne do_kvm_##n #define __KVM_HANDLER(area, h, n) \ do_kvm_##n: \ ld r10,area+EX_R10(r13); \ - stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \ + stw r9,HSTATE_SCRATCH1(r13); \ ld r9,area+EX_R9(r13); \ - std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \ + std r12,HSTATE_SCRATCH0(r13); \ li r12,n; \ b kvmppc_interrupt @@ -114,9 +114,9 @@ do_kvm_##n: \ cmpwi r10,KVM_GUEST_MODE_SKIP; \ ld r10,area+EX_R10(r13); \ beq 89f; \ - stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \ + stw r9,HSTATE_SCRATCH1(r13); \ ld r9,area+EX_R9(r13); \ - std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \ + std r12,HSTATE_SCRATCH0(r13); \ li r12,n; \ b kvmppc_interrupt; \ 89: mtocrf 0x80,r9; \ -- cgit v1.1 From de56a948b9182fbcf92cb8212f114de096c2d574 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Wed, 29 Jun 2011 00:21:34 +0000 Subject: KVM: PPC: Add support for Book3S processors in hypervisor mode This adds support for KVM running on 64-bit Book 3S processors, specifically POWER7, in hypervisor mode. Using hypervisor mode means that the guest can use the processor's supervisor mode. That means that the guest can execute privileged instructions and access privileged registers itself without trapping to the host. This gives excellent performance, but does mean that KVM cannot emulate a processor architecture other than the one that the hardware implements. This code assumes that the guest is running paravirtualized using the PAPR (Power Architecture Platform Requirements) interface, which is the interface that IBM's PowerVM hypervisor uses. That means that existing Linux distributions that run on IBM pSeries machines will also run under KVM without modification. In order to communicate the PAPR hypercalls to qemu, this adds a new KVM_EXIT_PAPR_HCALL exit code to include/linux/kvm.h. Currently the choice between book3s_hv support and book3s_pr support (i.e. the existing code, which runs the guest in user mode) has to be made at kernel configuration time, so a given kernel binary can only do one or the other. This new book3s_hv code doesn't support MMIO emulation at present. Since we are running paravirtualized guests, this isn't a serious restriction. With the guest running in supervisor mode, most exceptions go straight to the guest. We will never get data or instruction storage or segment interrupts, alignment interrupts, decrementer interrupts, program interrupts, single-step interrupts, etc., coming to the hypervisor from the guest. Therefore this introduces a new KVMTEST_NONHV macro for the exception entry path so that we don't have to do the KVM test on entry to those exception handlers. We do however get hypervisor decrementer, hypervisor data storage, hypervisor instruction storage, and hypervisor emulation assist interrupts, so we have to handle those. In hypervisor mode, real-mode accesses can access all of RAM, not just a limited amount. Therefore we put all the guest state in the vcpu.arch and use the shadow_vcpu in the PACA only for temporary scratch space. We allocate the vcpu with kzalloc rather than vzalloc, and we don't use anything in the kvmppc_vcpu_book3s struct, so we don't allocate it. We don't have a shared page with the guest, but we still need a kvm_vcpu_arch_shared struct to store the values of various registers, so we include one in the vcpu_arch struct. The POWER7 processor has a restriction that all threads in a core have to be in the same partition. MMU-on kernel code counts as a partition (partition 0), so we have to do a partition switch on every entry to and exit from the guest. At present we require the host and guest to run in single-thread mode because of this hardware restriction. This code allocates a hashed page table for the guest and initializes it with HPTEs for the guest's Virtual Real Memory Area (VRMA). We require that the guest memory is allocated using 16MB huge pages, in order to simplify the low-level memory management. This also means that we can get away without tracking paging activity in the host for now, since huge pages can't be paged or swapped. This also adds a few new exports needed by the book3s_hv code. Signed-off-by: Paul Mackerras Signed-off-by: Alexander Graf --- arch/powerpc/include/asm/exception-64s.h | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm/exception-64s.h') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 296c9b6..69435da 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -134,6 +134,17 @@ do_kvm_##n: \ #define KVM_HANDLER_SKIP(area, h, n) #endif +#ifdef CONFIG_KVM_BOOK3S_PR +#define KVMTEST_PR(n) __KVMTEST(n) +#define KVM_HANDLER_PR(area, h, n) __KVM_HANDLER(area, h, n) +#define KVM_HANDLER_PR_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n) + +#else +#define KVMTEST_PR(n) +#define KVM_HANDLER_PR(area, h, n) +#define KVM_HANDLER_PR_SKIP(area, h, n) +#endif + #define NOTEST(n) /* @@ -210,7 +221,7 @@ label##_pSeries: \ HMT_MEDIUM; \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \ - EXC_STD, KVMTEST, vec) + EXC_STD, KVMTEST_PR, vec) #define STD_EXCEPTION_HV(loc, vec, label) \ . = loc; \ @@ -227,8 +238,8 @@ label##_hv: \ beq masked_##h##interrupt #define _SOFTEN_TEST(h) __SOFTEN_TEST(h) -#define SOFTEN_TEST(vec) \ - KVMTEST(vec); \ +#define SOFTEN_TEST_PR(vec) \ + KVMTEST_PR(vec); \ _SOFTEN_TEST(EXC_STD) #define SOFTEN_TEST_HV(vec) \ @@ -248,7 +259,7 @@ label##_hv: \ .globl label##_pSeries; \ label##_pSeries: \ _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_TEST) + EXC_STD, SOFTEN_TEST_PR) #define MASKABLE_EXCEPTION_HV(loc, vec, label) \ . = loc; \ -- cgit v1.1 From 9e368f2915601cd5bc7f5fd638b58435b018bbd7 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Wed, 29 Jun 2011 00:40:08 +0000 Subject: KVM: PPC: book3s_hv: Add support for PPC970-family processors This adds support for running KVM guests in supervisor mode on those PPC970 processors that have a usable hypervisor mode. Unfortunately, Apple G5 machines have supervisor mode disabled (MSR[HV] is forced to 1), but the YDL PowerStation does have a usable hypervisor mode. There are several differences between the PPC970 and POWER7 in how guests are managed. These differences are accommodated using the CPU_FTR_ARCH_201 (PPC970) and CPU_FTR_ARCH_206 (POWER7) CPU feature bits. Notably, on PPC970: * The LPCR, LPID or RMOR registers don't exist, and the functions of those registers are provided by bits in HID4 and one bit in HID0. * External interrupts can be directed to the hypervisor, but unlike POWER7 they are masked by MSR[EE] in non-hypervisor modes and use SRR0/1 not HSRR0/1. * There is no virtual RMA (VRMA) mode; the guest must use an RMO (real mode offset) area. * The TLB entries are not tagged with the LPID, so it is necessary to flush the whole TLB on partition switch. Furthermore, when switching partitions we have to ensure that no other CPU is executing the tlbie or tlbsync instructions in either the old or the new partition, otherwise undefined behaviour can occur. * The PMU has 8 counters (PMC registers) rather than 6. * The DSCR, PURR, SPURR, AMR, AMOR, UAMOR registers don't exist. * The SLB has 64 entries rather than 32. * There is no mediated external interrupt facility, so if we switch to a guest that has a virtual external interrupt pending but the guest has MSR[EE] = 0, we have to arrange to have an interrupt pending for it so that we can get control back once it re-enables interrupts. We do that by sending ourselves an IPI with smp_send_reschedule after hard-disabling interrupts. Signed-off-by: Paul Mackerras Signed-off-by: Alexander Graf --- arch/powerpc/include/asm/exception-64s.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/powerpc/include/asm/exception-64s.h') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 69435da..8057f4f 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -246,6 +246,10 @@ label##_hv: \ KVMTEST(vec); \ _SOFTEN_TEST(EXC_HV) +#define SOFTEN_TEST_HV_201(vec) \ + KVMTEST(vec); \ + _SOFTEN_TEST(EXC_STD) + #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ HMT_MEDIUM; \ SET_SCRATCH0(r13); /* save r13 */ \ -- cgit v1.1 From c58ad1af9efe5fedc1e35362e53d4329a9e288e7 Mon Sep 17 00:00:00 2001 From: Michael Neuling Date: Mon, 16 Dec 2013 15:12:43 +1100 Subject: powerpc: Fix bad stack check in exception entry commit 90ff5d688e61f49f23545ffab6228bd7e87e6dc7 upstream. In EXCEPTION_PROLOG_COMMON() we check to see if the stack pointer (r1) is valid when coming from the kernel. If it's not valid, we die but with a nice oops message. Currently we allocate a stack frame (subtract INT_FRAME_SIZE) before we check to see if the stack pointer is negative. Unfortunately, this won't detect a bad stack where r1 is less than INT_FRAME_SIZE. This patch fixes the check to compare the modified r1 with -INT_FRAME_SIZE. With this, bad kernel stack pointers (including NULL pointers) are correctly detected again. Kudos to Paulus for finding this. Signed-off-by: Michael Neuling Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Ben Hutchings --- arch/powerpc/include/asm/exception-64s.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm/exception-64s.h') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 8057f4f..da85ad4 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -163,7 +163,7 @@ do_kvm_##n: \ subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ beq- 1f; \ ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ -1: cmpdi cr1,r1,0; /* check if r1 is in userspace */ \ +1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ blt+ cr1,3f; /* abort if it is */ \ li r1,(n); /* will be reloaded later */ \ sth r1,PACA_TRAP_SAVE(r13); \ -- cgit v1.1