AArch64: libm: Fix ARM64 fenv_t and refactor ARM64 libm implementation.

This patch fixes the ARM64 ABI for libm. fenv_t is now split in 32bit status
and 32bit control. This mirrors the AArch64 FPU control and status
registers (FPCR, FPSR).

The patch also refactors the libm implementation for ARM64 into a finer
grained control over the FPU registers.

Bionic-benchmarks has been expanded with 3 more benchmarks for floating
point operations. The new libm implementation for ARM64 performs better
over all the math benchmarks available.

Change-Id: I2a7f81d6b4e55c91f8a63a4c69614fc8b1bcf2db
Signed-off-by: Serban Constantinescu <serban.constantinescu@arm.com>

1 files changed, 126 insertions, 72 deletions

diff --git a/libm/arm64/fenv.c b/libm/arm64/fenv.c
index 9db21ef..ce560a7 100644
--- a/libm/arm64/fenv.c
+++ b/libm/arm64/fenv.c
@@ -28,114 +28,168 @@
 
 #include <fenv.h>
 
-/*
- * Hopefully the system ID byte is immutable, so it's valid to use
- * this as a default environment.
- */
-const fenv_t __fe_dfl_env = 0;
+#define FPCR_EXCEPT_SHIFT 8
+#define FPCR_EXCEPT_MASK  (FE_ALL_EXCEPT << FPCR_EXCEPT_SHIFT)
+
+#define FPCR_RMODE_SHIFT 22
+
+const fenv_t __fe_dfl_env = { 0 /* control */, 0 /* status */};
+
+typedef __uint32_t fpu_control_t;   // FPCR, Floating-point Control Register.
+typedef __uint32_t fpu_status_t;    // FPSR, Floating-point Status Register.
+
+#define __get_fpcr(__fpcr) __asm__ __volatile__("mrs %0,fpcr" : "=r" (__fpcr))
+#define __get_fpsr(__fpsr) __asm__ __volatile__("mrs %0,fpsr" : "=r" (__fpsr))
+#define __set_fpcr(__fpcr) __asm__ __volatile__("msr fpcr,%0" : :"ri" (__fpcr))
+#define __set_fpsr(__fpsr) __asm__ __volatile__("msr fpsr,%0" : :"ri" (__fpsr))
 
-int fegetenv(fenv_t* __envp) {
-  fenv_t _fpcr, _fpsr;
-  __asm__ __volatile__("mrs %0,fpcr" : "=r" (_fpcr));
-  __asm__ __volatile__("mrs %0,fpsr" : "=r" (_fpsr));
-  *__envp = (_fpcr | _fpsr);
+int fegetenv(fenv_t* envp) {
+  __get_fpcr(envp->__control);
+  __get_fpsr(envp->__status);
   return 0;
 }
 
-int fesetenv(const fenv_t* __envp) {
-  fenv_t _fpcr = (*__envp & FPCR_MASK);
-  fenv_t _fpsr = (*__envp & FPSR_MASK);
-  __asm__ __volatile__("msr fpcr,%0" : :"ri" (_fpcr));
-  __asm__ __volatile__("msr fpsr,%0" : :"ri" (_fpsr));
+int fesetenv(const fenv_t* envp) {
+  fpu_control_t fpcr;
+
+  __get_fpcr(fpcr);
+  if (envp->__control != fpcr) {
+    __set_fpcr(envp->__control);
+  }
+  __set_fpsr(envp->__status);
   return 0;
 }
 
-int feclearexcept(int __excepts) {
-  fexcept_t __fpscr;
-  fegetenv(&__fpscr);
-  __fpscr &= ~__excepts;
-  fesetenv(&__fpscr);
+int feclearexcept(int excepts) {
+  fpu_status_t fpsr;
+
+  excepts &= FE_ALL_EXCEPT;
+  __get_fpsr(fpsr);
+  fpsr &= ~excepts;
+  __set_fpsr(fpsr);
   return 0;
 }
 
-int fegetexceptflag(fexcept_t* __flagp, int __excepts) {
-  fexcept_t __fpscr;
-  fegetenv(&__fpscr);
-  *__flagp = __fpscr & __excepts;
+int fegetexceptflag(fexcept_t* flagp, int excepts) {
+  fpu_status_t fpsr;
+
+  excepts &= FE_ALL_EXCEPT;
+  __get_fpsr(fpsr);
+  *flagp = fpsr & excepts;
   return 0;
 }
 
-int fesetexceptflag(const fexcept_t* __flagp, int __excepts) {
-  fexcept_t __fpscr;
-  fegetenv(&__fpscr);
-  __fpscr &= ~__excepts;
-  __fpscr |= *__flagp & __excepts;
-  fesetenv(&__fpscr);
+int fesetexceptflag(const fexcept_t* flagp, int excepts) {
+  fpu_status_t fpsr;
+
+  excepts &= FE_ALL_EXCEPT;
+  __get_fpsr(fpsr);
+  fpsr &= ~excepts;
+  fpsr |= *flagp & excepts;
+  __set_fpsr(fpsr);
   return 0;
 }
 
-int feraiseexcept(int __excepts) {
-  fexcept_t __ex = __excepts;
-  fesetexceptflag(&__ex, __excepts);
+int feraiseexcept(int excepts) {
+  fexcept_t ex = excepts;
+
+  fesetexceptflag(&ex, excepts);
   return 0;
 }
 
-int fetestexcept(int __excepts) {
-  fexcept_t __fpscr;
-  fegetenv(&__fpscr);
-  return (__fpscr & __excepts);
+int fetestexcept(int excepts) {
+  fpu_status_t fpsr;
+
+  excepts &= FE_ALL_EXCEPT;
+  __get_fpsr(fpsr);
+  return (fpsr & excepts);
 }
 
 int fegetround(void) {
-  fenv_t _fpscr;
-  fegetenv(&_fpscr);
-  return ((_fpscr >> _FPSCR_RMODE_SHIFT) & 0x3);
+  fpu_control_t fpcr;
+
+  __get_fpcr(fpcr);
+  return ((fpcr >> FPCR_RMODE_SHIFT) & FE_TOWARDZERO);
 }
 
-int fesetround(int __round) {
-  fenv_t _fpscr;
-  fegetenv(&_fpscr);
-  _fpscr &= ~(0x3 << _FPSCR_RMODE_SHIFT);
-  _fpscr |= (__round << _FPSCR_RMODE_SHIFT);
-  fesetenv(&_fpscr);
+int fesetround(int round) {
+  fpu_control_t fpcr, new_fpcr;
+
+  round &= FE_TOWARDZERO;
+  __get_fpcr(fpcr);
+  new_fpcr = fpcr & ~(FE_TOWARDZERO << FPCR_RMODE_SHIFT);
+  new_fpcr |= (round << FPCR_RMODE_SHIFT);
+  if (new_fpcr != fpcr) {
+    __set_fpcr(new_fpcr);
+  }
   return 0;
 }
 
-int feholdexcept(fenv_t* __envp) {
-  fenv_t __env;
-  fegetenv(&__env);
-  *__envp = __env;
-  __env &= ~(FE_ALL_EXCEPT | _FPSCR_ENABLE_MASK);
-  fesetenv(&__env);
+int feholdexcept(fenv_t* envp) {
+  fenv_t env;
+  fpu_status_t fpsr;
+  fpu_control_t fpcr, new_fpcr;
+
+  __get_fpsr(fpsr);
+  __get_fpcr(fpcr);
+  env.__status = fpsr;
+  env.__control = fpcr;
+  *envp = env;
+
+  // Set exceptions to untrapped.
+  new_fpcr = fpcr & ~(FE_ALL_EXCEPT << FPCR_EXCEPT_SHIFT);
+  if (new_fpcr != fpcr) {
+    __set_fpcr(new_fpcr);
+  }
+
+  // Clear all exceptions.
+  fpsr &= ~FE_ALL_EXCEPT;
+  __set_fpsr(fpsr);
   return 0;
 }
 
-int feupdateenv(const fenv_t* __envp) {
-  fexcept_t __fpscr;
-  fegetenv(&__fpscr);
-  fesetenv(__envp);
-  feraiseexcept(__fpscr & FE_ALL_EXCEPT);
+int feupdateenv(const fenv_t* envp) {
+  fpu_status_t fpsr;
+  fpu_control_t fpcr;
+
+  // Set FPU Control register.
+  __get_fpcr(fpcr);
+  if (envp->__control != fpcr) {
+    __set_fpcr(envp->__control);
+  }
+
+  // Set FPU Status register to status | currently raised exceptions.
+  __get_fpsr(fpsr);
+  fpsr = envp->__status | (fpsr & FE_ALL_EXCEPT);
+  __set_fpsr(fpsr);
   return 0;
 }
 
-int feenableexcept(int __mask) {
-  fenv_t __old_fpscr, __new_fpscr;
-  fegetenv(&__old_fpscr);
-  __new_fpscr = __old_fpscr | (__mask & FE_ALL_EXCEPT) << _FPSCR_ENABLE_SHIFT;
-  fesetenv(&__new_fpscr);
-  return ((__old_fpscr >> _FPSCR_ENABLE_SHIFT) & FE_ALL_EXCEPT);
+int feenableexcept(int mask) {
+  fpu_control_t old_fpcr, new_fpcr;
+
+  __get_fpcr(old_fpcr);
+  new_fpcr = old_fpcr | ((mask & FE_ALL_EXCEPT) << FPCR_EXCEPT_SHIFT);
+  if (new_fpcr != old_fpcr) {
+    __set_fpcr(new_fpcr);
+  }
+  return ((old_fpcr >> FPCR_EXCEPT_SHIFT) & FE_ALL_EXCEPT);
 }
 
-int fedisableexcept(int __mask) {
-  fenv_t __old_fpscr, __new_fpscr;
-  fegetenv(&__old_fpscr);
-  __new_fpscr = __old_fpscr & ~((__mask & FE_ALL_EXCEPT) << _FPSCR_ENABLE_SHIFT);
-  fesetenv(&__new_fpscr);
-  return ((__old_fpscr >> _FPSCR_ENABLE_SHIFT) & FE_ALL_EXCEPT);
+int fedisableexcept(int mask) {
+  fpu_control_t old_fpcr, new_fpcr;
+
+  __get_fpcr(old_fpcr);
+  new_fpcr = old_fpcr & ~((mask & FE_ALL_EXCEPT) << FPCR_EXCEPT_SHIFT);
+  if (new_fpcr != old_fpcr) {
+    __set_fpcr(new_fpcr);
+  }
+  return ((old_fpcr >> FPCR_EXCEPT_SHIFT) & FE_ALL_EXCEPT);
 }
 
 int fegetexcept(void) {
-  fenv_t __fpscr;
-  fegetenv(&__fpscr);
-  return ((__fpscr & _FPSCR_ENABLE_MASK) >> _FPSCR_ENABLE_SHIFT);
+  fpu_control_t fpcr;
+
+  __get_fpcr(fpcr);
+  return ((fpcr & FPCR_EXCEPT_MASK) >> FPCR_EXCEPT_SHIFT);
 }