bionic: Optimizations for gettimeofday and clock_gettime.

Use the kernel user helper feature to calculate the results
of gettimeofday and clock_gettime without actually calling
into kernel space. If the user helper patches have not been
applied to the kernel, the regular system calls are used as
a fallback.

Change-Id: I3aebc6ac19ab4743725648a1a279819e624cc5c4

6 files changed, 357 insertions, 0 deletions

diff --git a/libc/Android.mk b/libc/Android.mk
index d6a5aa5..245f6a4 100644
--- a/libc/Android.mk
+++ b/libc/Android.mk
@@ -412,6 +412,18 @@ ifneq (, $(filter true,$(TARGET_USE_KRAIT_BIONIC_OPTIMIZATION) $(TARGET_USE_SPAR
  endif # !TARGET_USE_KRAIT_BIONIC_OPTIMIZATION
 endif # !TARGET_USE_SCORPION_BIONIC_OPTIMIZATION
 
+# If the kernel supports kernel user helpers for gettimeofday, use
+# that instead.
+ifeq ($(KERNEL_HAS_GETTIMEOFDAY_HELPER),true)
+  libc_common_src_files := $(filter-out arch-arm/syscalls/gettimeofday.S,$(libc_common_src_files))
+  libc_common_src_files := $(filter-out arch-arm/syscalls/clock_gettime.S,$(libc_common_src_files))
+  libc_common_src_files += \
+	arch-arm/bionic/gettimeofday.c \
+	arch-arm/bionic/gettimeofday_syscall.S \
+	arch-arm/bionic/clock_gettime.c \
+	arch-arm/bionic/clock_gettime_syscall.S
+endif # KERNEL_HAS_GETTIMEOFDAY_HELPER
+
 # These files need to be arm so that gdbserver
 # can set breakpoints in them without messing
 # up any thumb code.
diff --git a/libc/arch-arm/bionic/clock_gettime.c b/libc/arch-arm/bionic/clock_gettime.c
new file mode 100644
index 0000000..c2917b0
--- /dev/null
+++ b/libc/arch-arm/bionic/clock_gettime.c
@@ -0,0 +1,94 @@
+/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *    * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following
+ *     disclaimer in the documentation and/or other materials provided
+ *     with the distribution.
+ *   * Neither the name of The Linux Foundation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <time.h>
+#include <sys/time.h>
+#include <machine/cpu-features.h>
+#include <machine/kernel_user_helper.h>
+
+int clock_gettime(int clk_id, struct timespec *tp)
+{
+	unsigned prelock, postlock;
+
+	/*
+	 * Check if the offset in the kernel user helper page has
+	 * the flag set appropriately to show that this feature is
+	 * enabled in the kernel.  If not, default to the original
+	 * clock_gettime system call.
+	 *
+	 * Also, if this is anything other than CLOCK_MONOTONIC, route
+	 * to the original system call as well.
+	 */
+	if ((__kuser_gtod_feature != __kuser_gtod_feature_flag) ||
+		(clk_id != CLOCK_MONOTONIC))
+		return clock_gettime_syscall(clk_id, tp);
+
+	if (tp) {
+		struct gtod_t dgtod;
+		uint32_t nscount, cycleoffset;
+		uint32_t mono_sec, mono_nsec;
+		uint64_t cycle_delta;
+
+		do {
+			prelock = __kuser_gtod_seqnum;
+
+			dgtod.cycle_last = __kuser_gtod_cycle_last;
+			dgtod.mask = __kuser_gtod_mask;
+			dgtod.mult = __kuser_gtod_mult;
+			dgtod.shift = __kuser_gtod_shift;
+			dgtod.tv_sec = __kuser_gtod_tv_sec;
+			dgtod.tv_nsec = __kuser_gtod_tv_nsec;
+
+			mono_sec = __kuser_gtod_wtm_tv_sec;
+			mono_nsec = __kuser_gtod_wtm_tv_nsec;
+
+			cycleoffset = __kuser_gtod_offset;
+			cycleoffset += __kuser_gtod_cycle_base;
+			nscount = *(uint32_t *)cycleoffset;
+
+			postlock = __kuser_gtod_seqnum;
+		} while (prelock != postlock);
+
+		cycle_delta = (nscount - dgtod.cycle_last) & dgtod.mask;
+		dgtod.tv_nsec += (cycle_delta * dgtod.mult) >> dgtod.shift;
+		dgtod.tv_sec += mono_sec;
+		dgtod.tv_nsec += mono_nsec;
+		while (dgtod.tv_nsec >= NSEC_PER_SEC) {
+			dgtod.tv_sec += 1;
+			dgtod.tv_nsec -= NSEC_PER_SEC;
+		}
+
+		tp->tv_sec = dgtod.tv_sec;
+		tp->tv_nsec = dgtod.tv_nsec;
+	}
+
+	return 0;
+}
diff --git a/libc/arch-arm/bionic/clock_gettime_syscall.S b/libc/arch-arm/bionic/clock_gettime_syscall.S
new file mode 100644
index 0000000..0b3078a
--- /dev/null
+++ b/libc/arch-arm/bionic/clock_gettime_syscall.S
@@ -0,0 +1,42 @@
+/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *    * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following
+ *     disclaimer in the documentation and/or other materials provided
+ *     with the distribution.
+ *   * Neither the name of The Linux Foundation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <machine/asm.h>
+#include <sys/linux-syscalls.h>
+
+ENTRY(clock_gettime_syscall)
+	.save   {r4, r7}
+	stmfd   sp!, {r4, r7}
+	ldr     r7, =__NR_clock_gettime
+	swi     #0
+	movs    r0, r0
+	ldmfd   sp!, {r4, r7}
+	bmi     __set_syscall_errno
+	bx      lr
+END(clock_gettime_syscall)
diff --git a/libc/arch-arm/bionic/gettimeofday.c b/libc/arch-arm/bionic/gettimeofday.c
new file mode 100644
index 0000000..780d6e8
--- /dev/null
+++ b/libc/arch-arm/bionic/gettimeofday.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *    * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following
+ *     disclaimer in the documentation and/or other materials provided
+ *     with the distribution.
+ *   * Neither the name of The Linux Foundation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <sys/time.h>
+#include <machine/cpu-features.h>
+#include <machine/kernel_user_helper.h>
+
+int gettimeofday(struct timeval *tv, struct timezone *tz)
+{
+	unsigned prelock, postlock;
+
+	/*
+	 * Check if the offset in the kernel user helper page has
+	 * the flag set appropriately to show that this feature is
+	 * enabled in the kernel.  If not, default to the original
+	 * gettimeofday system call.
+	 */
+	if (__kuser_gtod_feature != __kuser_gtod_feature_flag)
+		return gettimeofday_syscall(tv, tz);
+
+	if (tv) {
+		struct gtod_t dgtod;
+		uint32_t nscount, cycleoffset;
+		uint64_t cycle_delta;
+		uint32_t tmp = 0;
+
+		do {
+			prelock = __kuser_gtod_seqnum;
+
+			dgtod.cycle_last = __kuser_gtod_cycle_last;
+			dgtod.mask = __kuser_gtod_mask;
+			dgtod.mult = __kuser_gtod_mult;
+			dgtod.shift = __kuser_gtod_shift;
+			dgtod.tv_sec = __kuser_gtod_tv_sec;
+			dgtod.tv_nsec = __kuser_gtod_tv_nsec;
+
+			cycleoffset = __kuser_gtod_offset;
+			cycleoffset += __kuser_gtod_cycle_base;
+			nscount = *(uint32_t *)cycleoffset;
+
+			postlock = __kuser_gtod_seqnum;
+		} while (prelock != postlock);
+
+		cycle_delta = (nscount - dgtod.cycle_last) & dgtod.mask;
+		dgtod.tv_nsec += (cycle_delta * dgtod.mult) >> dgtod.shift;
+		while (dgtod.tv_nsec >= NSEC_PER_SEC) {
+			dgtod.tv_sec += 1;
+			dgtod.tv_nsec -= NSEC_PER_SEC;
+		}
+
+		tv->tv_sec = dgtod.tv_sec;
+		asm("   movw   %[tmp], #0x4dd3\n\t"
+		    "   movt   %[tmp], #0x1062\n\t"
+		    "   umull  %[tmp], %[x], %[y], %[tmp]\n\t"
+		    "   lsr    %[x], %[x], #6\n\t" :
+			[x] "=r" (tv->tv_usec) :
+			[y] "r" (dgtod.tv_nsec), [tmp] "r" (tmp)
+			: );
+	}
+
+	if (tz) {
+		do {
+			prelock = __kuser_gtod_seqnum;
+			tz->tz_minuteswest = __kuser_gtod_tz_minw;
+			tz->tz_dsttime = __kuser_gtod_tz_dst;
+			postlock = __kuser_gtod_seqnum;
+		} while (prelock != postlock);
+	}
+
+	return 0;
+}
diff --git a/libc/arch-arm/bionic/gettimeofday_syscall.S b/libc/arch-arm/bionic/gettimeofday_syscall.S
new file mode 100644
index 0000000..3a945e2
--- /dev/null
+++ b/libc/arch-arm/bionic/gettimeofday_syscall.S
@@ -0,0 +1,42 @@
+/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *    * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following
+ *     disclaimer in the documentation and/or other materials provided
+ *     with the distribution.
+ *   * Neither the name of The Linux Foundation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <machine/asm.h>
+#include <sys/linux-syscalls.h>
+
+ENTRY(gettimeofday_syscall)
+	.save   {r4, r7}
+	stmfd   sp!, {r4, r7}
+	ldr     r7, =__NR_gettimeofday
+	swi     #0
+	movs    r0, r0
+	ldmfd   sp!, {r4, r7}
+	bmi     __set_syscall_errno
+	bx      lr
+END(gettimeofday_syscall)
diff --git a/libc/arch-arm/include/machine/kernel_user_helper.h b/libc/arch-arm/include/machine/kernel_user_helper.h
new file mode 100644
index 0000000..8836c50
--- /dev/null
+++ b/libc/arch-arm/include/machine/kernel_user_helper.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *    * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following
+ *     disclaimer in the documentation and/or other materials provided
+ *     with the distribution.
+ *   * Neither the name of The Linux Foundation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef _ARCH_ARM_KERNEL_USER_HELPER_H
+#define _ARCH_ARM_KERNEL_USER_HELPER_H
+
+extern int clock_gettime_syscall(int clk_id, struct timespec *tp);
+extern int gettimeofday_syscall(struct timeval *tv, struct timezone *tz);
+
+#define __kuser_gtod_base        (*(int32_t *)0xffff0f40)
+#define __kuser_gtod_cycle_last  (*(int32_t *)0xffff0f40)
+#define __kuser_gtod_mask        (*(int32_t *)0xffff0f48)
+#define __kuser_gtod_mult        (*(int32_t *)0xffff0f50)
+#define __kuser_gtod_shift       (*(int32_t *)0xffff0f54)
+#define __kuser_gtod_tv_sec      (*(int32_t *)0xffff0f58)
+#define __kuser_gtod_tv_nsec     (*(int32_t *)0xffff0f5c)
+
+#define __kuser_gtod_seqnum      (*(int32_t *)0xffff0f28)
+#define __kuser_gtod_offset      (*(int32_t *)0xffff0f30)
+#define __kuser_gtod_cycle_base               0xfffef000
+#define __kuser_gtod_feature     (*(int32_t *)0xffff0f34)
+#define __kuser_gtod_feature_flag             0xffff0f20
+
+#define __kuser_gtod_wtm_tv_sec  (*(int32_t *)0xffff0f38)
+#define __kuser_gtod_wtm_tv_nsec (*(int32_t *)0xffff0f3c)
+
+#define __kuser_gtod_timezone    (*(int32_t *)0xffff0f20)
+#define __kuser_gtod_tz_minw     (*(int32_t *)0xffff0f20)
+#define __kuser_gtod_tz_dst      (*(int32_t *)0xffff0f24)
+
+struct gtod_t {
+	uint64_t  cycle_last;
+	uint64_t  mask;
+	uint32_t  mult;
+	uint32_t  shift;
+	uint32_t  tv_sec;
+	uint32_t  tv_nsec;
+};
+
+#define NSEC_PER_SEC	1000000000L
+
+#endif