diff options
Diffstat (limited to 'libc/arch-arm64')
| -rw-r--r-- | libc/arch-arm64/arm64.mk | 31 | ||||
| -rw-r--r-- | libc/arch-arm64/bionic/__rt_sigreturn.S | 34 | ||||
| -rw-r--r-- | libc/arch-arm64/denver64/bionic/memcpy.S | 248 | ||||
| -rw-r--r-- | libc/arch-arm64/denver64/bionic/memcpy_base.S | 199 | ||||
| -rw-r--r-- | libc/arch-arm64/generic-neon/generic-neon.mk | 13 | ||||
| -rw-r--r-- | libc/arch-arm64/generic/bionic/memcpy.S | 227 | ||||
| -rw-r--r-- | libc/arch-arm64/generic/bionic/memcpy_base.S (renamed from libc/arch-arm64/generic-neon/bionic/memcpy.S) | 54 | ||||
| -rw-r--r-- | libc/arch-arm64/include/machine/elf_machdep.h | 1 |
8 files changed, 351 insertions, 456 deletions
diff --git a/libc/arch-arm64/arm64.mk b/libc/arch-arm64/arm64.mk index 6c4f6a6..bb6ca63 100644 --- a/libc/arch-arm64/arm64.mk +++ b/libc/arch-arm64/arm64.mk @@ -1,8 +1,17 @@ -# arm64 specific configs +# 64-bit arm. -libc_common_src_files_arm64 := \ +# +# Default implementations of functions that are commonly optimized. +# + +libc_bionic_src_files_arm64 += \ + bionic/__memset_chk.cpp \ + bionic/__strcpy_chk.cpp \ + bionic/__strcat_chk.cpp \ bionic/memrchr.c \ bionic/strrchr.cpp \ + +libc_freebsd_src_files_arm64 += \ upstream-freebsd/lib/libc/string/wcscat.c \ upstream-freebsd/lib/libc/string/wcschr.c \ upstream-freebsd/lib/libc/string/wcscmp.c \ @@ -10,6 +19,8 @@ libc_common_src_files_arm64 := \ upstream-freebsd/lib/libc/string/wcslen.c \ upstream-freebsd/lib/libc/string/wcsrchr.c \ upstream-freebsd/lib/libc/string/wmemcmp.c \ + +libc_openbsd_src_files_arm64 += \ upstream-openbsd/lib/libc/string/stpncpy.c \ upstream-openbsd/lib/libc/string/strcat.c \ upstream-openbsd/lib/libc/string/strlcat.c \ @@ -17,19 +28,13 @@ libc_common_src_files_arm64 := \ upstream-openbsd/lib/libc/string/strncat.c \ upstream-openbsd/lib/libc/string/strncpy.c \ -# Fortify implementations of libc functions. -libc_common_src_files_arm64 += \ - bionic/__memcpy_chk.cpp \ - bionic/__memset_chk.cpp \ - bionic/__strcpy_chk.cpp \ - bionic/__strcat_chk.cpp \ +# +# Inherently architecture-specific code. +# -########################################## -### CPU specific source files -libc_bionic_src_files_arm64 := \ +libc_bionic_src_files_arm64 += \ arch-arm64/bionic/__bionic_clone.S \ arch-arm64/bionic/_exit_with_stack_teardown.S \ - arch-arm64/bionic/__rt_sigreturn.S \ arch-arm64/bionic/_setjmp.S \ arch-arm64/bionic/setjmp.S \ arch-arm64/bionic/__set_tls.c \ @@ -53,7 +58,7 @@ ifeq ($(strip $(TARGET_CPU_VARIANT)),) endif cpu_variant_mk := $(LOCAL_PATH)/arch-arm64/$(TARGET_CPU_VARIANT)/$(TARGET_CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) -$(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are generic, generic-neon, denver64. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") +$(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are generic, denver64. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif include $(cpu_variant_mk) libc_common_additional_dependencies += $(cpu_variank_mk) diff --git a/libc/arch-arm64/bionic/__rt_sigreturn.S b/libc/arch-arm64/bionic/__rt_sigreturn.S deleted file mode 100644 index 8fb6f0c..0000000 --- a/libc/arch-arm64/bionic/__rt_sigreturn.S +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <private/bionic_asm.h> - -ENTRY_PRIVATE(__rt_sigreturn) - mov x8, __NR_rt_sigreturn - svc #0 -END(__rt_sigreturn) diff --git a/libc/arch-arm64/denver64/bionic/memcpy.S b/libc/arch-arm64/denver64/bionic/memcpy.S index 700f0d0..85129fe 100644 --- a/libc/arch-arm64/denver64/bionic/memcpy.S +++ b/libc/arch-arm64/denver64/bionic/memcpy.S @@ -1,205 +1,63 @@ -/* Copyright (c) 2012, Linaro Limited - All rights reserved. - Copyright (c) 2014, NVIDIA Corporation. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the Linaro nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* Assumptions: +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. * - * denver, ARMv8-a, AArch64 - * Unaligned accesses + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ -#include <private/bionic_asm.h> +// Prototype: void *memcpy (void *dst, const void *src, size_t count). -#define dstin x0 -#define src x1 -#define count x2 -#define tmp1 x3 -#define tmp1w w3 -#define tmp2 x4 -#define tmp2w w4 -#define tmp3 x5 -#define tmp3w w5 -#define dst x6 +#include <private/bionic_asm.h> +#include <private/libc_events.h> -#define A_l x7 -#define A_h x8 -#define B_l x9 -#define B_h x10 -#define C_l x11 -#define C_h x12 -#define D_l x13 -#define D_h x14 +ENTRY(__memcpy_chk) + cmp x2, x3 + b.hi __memcpy_chk_fail -#define QA_l q0 -#define QA_h q1 -#define QB_l q2 -#define QB_h q3 + // Fall through to memcpy... +END(__memcpy_chk) ENTRY(memcpy) - - mov dst, dstin - cmp count, #64 - b.ge .Lcpy_not_short - cmp count, #15 - b.le .Ltail15tiny - - /* Deal with small copies quickly by dropping straight into the - * exit block. */ -.Ltail63: - /* Copy up to 48 bytes of data. At this point we only need the - * bottom 6 bits of count to be accurate. */ - ands tmp1, count, #0x30 - b.eq .Ltail15 - add dst, dst, tmp1 - add src, src, tmp1 - cmp tmp1w, #0x20 - b.eq 1f - b.lt 2f - ldp A_l, A_h, [src, #-48] - stp A_l, A_h, [dst, #-48] -1: - ldp A_l, A_h, [src, #-32] - stp A_l, A_h, [dst, #-32] -2: - ldp A_l, A_h, [src, #-16] - stp A_l, A_h, [dst, #-16] - -.Ltail15: - ands count, count, #15 - beq 1f - add src, src, count - ldp A_l, A_h, [src, #-16] - add dst, dst, count - stp A_l, A_h, [dst, #-16] -1: - ret - -.Ltail15tiny: - /* Copy up to 15 bytes of data. Does not assume additional data - being copied. */ - tbz count, #3, 1f - ldr tmp1, [src], #8 - str tmp1, [dst], #8 -1: - tbz count, #2, 1f - ldr tmp1w, [src], #4 - str tmp1w, [dst], #4 -1: - tbz count, #1, 1f - ldrh tmp1w, [src], #2 - strh tmp1w, [dst], #2 -1: - tbz count, #0, 1f - ldrb tmp1w, [src] - strb tmp1w, [dst] -1: - ret - -.Lcpy_not_short: - /* We don't much care about the alignment of DST, but we want SRC - * to be 128-bit (16 byte) aligned so that we don't cross cache line - * boundaries on both loads and stores. */ - neg tmp2, src - ands tmp2, tmp2, #15 /* Bytes to reach alignment. */ - b.eq 2f - sub count, count, tmp2 - /* Copy more data than needed; it's faster than jumping - * around copying sub-Quadword quantities. We know that - * it can't overrun. */ - ldp A_l, A_h, [src] - add src, src, tmp2 - stp A_l, A_h, [dst] - add dst, dst, tmp2 - /* There may be less than 63 bytes to go now. */ - cmp count, #63 - b.le .Ltail63 -2: - subs count, count, #128 - b.ge .Lcpy_body_large - /* Less than 128 bytes to copy, so handle 64 here and then jump - * to the tail. */ - ldp QA_l, QA_h, [src] - ldp QB_l, QB_h, [src, #32] - stp QA_l, QA_h, [dst] - stp QB_l, QB_h, [dst, #32] - tst count, #0x3f - add src, src, #64 - add dst, dst, #64 - b.ne .Ltail63 - ret - - /* Critical loop. Start at a new cache line boundary. Assuming - * 64 bytes per line this ensures the entire loop is in one line. */ - .p2align 6 -.Lcpy_body_large: - cmp count, 65536 - bhi .Lcpy_body_huge - /* There are at least 128 bytes to copy. */ - ldp QA_l, QA_h, [src, #0] - sub dst, dst, #32 /* Pre-bias. */ - ldp QB_l, QB_h, [src, #32]! /* src += 64 - Pre-bias. */ -1: - stp QA_l, QA_h, [dst, #32] - ldp QA_l, QA_h, [src, #32] - stp QB_l, QB_h, [dst, #64]! - ldp QB_l, QB_h, [src, #64]! - - subs count, count, #64 - b.ge 1b - - stp QA_l, QA_h, [dst, #32] - stp QB_l, QB_h, [dst, #64] - add src, src, #32 - add dst, dst, #64 + 32 - tst count, #0x3f - b.ne .Ltail63 - ret -.Lcpy_body_huge: - /* There are at least 128 bytes to copy. */ - ldp QA_l, QA_h, [src, #0] - sub dst, dst, #32 /* Pre-bias. */ - ldp QB_l, QB_h, [src, #32]! -1: - stnp QA_l, QA_h, [dst, #32] - stnp QB_l, QB_h, [dst, #64] - ldp QA_l, QA_h, [src, #32] - ldp QB_l, QB_h, [src, #64]! - add dst, dst, #64 - - subs count, count, #64 - b.ge 1b - - stnp QA_l, QA_h, [dst, #32] - stnp QB_l, QB_h, [dst, #64] - add src, src, #32 - add dst, dst, #64 + 32 - tst count, #0x3f - b.ne .Ltail63 - ret - + #include "memcpy_base.S" END(memcpy) + +ENTRY_PRIVATE(__memcpy_chk_fail) + // Preserve for accurate backtrace. + stp x29, x30, [sp, -16]! + .cfi_def_cfa_offset 16 + .cfi_rel_offset x29, 0 + .cfi_rel_offset x30, 8 + + adrp x0, error_string + add x0, x0, :lo12:error_string + ldr x1, error_code + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +END(__memcpy_chk_fail) + + .data + .align 2 +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm64/denver64/bionic/memcpy_base.S b/libc/arch-arm64/denver64/bionic/memcpy_base.S new file mode 100644 index 0000000..3d7e9dd --- /dev/null +++ b/libc/arch-arm64/denver64/bionic/memcpy_base.S @@ -0,0 +1,199 @@ +/* Copyright (c) 2012, Linaro Limited + All rights reserved. + Copyright (c) 2014, NVIDIA Corporation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Linaro nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Assumptions: + * + * denver, ARMv8-a, AArch64 + * Unaligned accesses + * + */ + +#define dstin x0 +#define src x1 +#define count x2 +#define tmp1 x3 +#define tmp1w w3 +#define tmp2 x4 +#define tmp2w w4 +#define tmp3 x5 +#define tmp3w w5 +#define dst x6 + +#define A_l x7 +#define A_h x8 +#define B_l x9 +#define B_h x10 +#define C_l x11 +#define C_h x12 +#define D_l x13 +#define D_h x14 + +#define QA_l q0 +#define QA_h q1 +#define QB_l q2 +#define QB_h q3 + + mov dst, dstin + cmp count, #64 + b.ge .Lcpy_not_short + cmp count, #15 + b.le .Ltail15tiny + + /* Deal with small copies quickly by dropping straight into the + * exit block. */ +.Ltail63: + /* Copy up to 48 bytes of data. At this point we only need the + * bottom 6 bits of count to be accurate. */ + ands tmp1, count, #0x30 + b.eq .Ltail15 + add dst, dst, tmp1 + add src, src, tmp1 + cmp tmp1w, #0x20 + b.eq 1f + b.lt 2f + ldp A_l, A_h, [src, #-48] + stp A_l, A_h, [dst, #-48] +1: + ldp A_l, A_h, [src, #-32] + stp A_l, A_h, [dst, #-32] +2: + ldp A_l, A_h, [src, #-16] + stp A_l, A_h, [dst, #-16] + +.Ltail15: + ands count, count, #15 + beq 1f + add src, src, count + ldp A_l, A_h, [src, #-16] + add dst, dst, count + stp A_l, A_h, [dst, #-16] +1: + ret + +.Ltail15tiny: + /* Copy up to 15 bytes of data. Does not assume additional data + being copied. */ + tbz count, #3, 1f + ldr tmp1, [src], #8 + str tmp1, [dst], #8 +1: + tbz count, #2, 1f + ldr tmp1w, [src], #4 + str tmp1w, [dst], #4 +1: + tbz count, #1, 1f + ldrh tmp1w, [src], #2 + strh tmp1w, [dst], #2 +1: + tbz count, #0, 1f + ldrb tmp1w, [src] + strb tmp1w, [dst] +1: + ret + +.Lcpy_not_short: + /* We don't much care about the alignment of DST, but we want SRC + * to be 128-bit (16 byte) aligned so that we don't cross cache line + * boundaries on both loads and stores. */ + neg tmp2, src + ands tmp2, tmp2, #15 /* Bytes to reach alignment. */ + b.eq 2f + sub count, count, tmp2 + /* Copy more data than needed; it's faster than jumping + * around copying sub-Quadword quantities. We know that + * it can't overrun. */ + ldp A_l, A_h, [src] + add src, src, tmp2 + stp A_l, A_h, [dst] + add dst, dst, tmp2 + /* There may be less than 63 bytes to go now. */ + cmp count, #63 + b.le .Ltail63 +2: + subs count, count, #128 + b.ge .Lcpy_body_large + /* Less than 128 bytes to copy, so handle 64 here and then jump + * to the tail. */ + ldp QA_l, QA_h, [src] + ldp QB_l, QB_h, [src, #32] + stp QA_l, QA_h, [dst] + stp QB_l, QB_h, [dst, #32] + tst count, #0x3f + add src, src, #64 + add dst, dst, #64 + b.ne .Ltail63 + ret + + /* Critical loop. Start at a new cache line boundary. Assuming + * 64 bytes per line this ensures the entire loop is in one line. */ + .p2align 6 +.Lcpy_body_large: + cmp count, 65536 + bhi .Lcpy_body_huge + /* There are at least 128 bytes to copy. */ + ldp QA_l, QA_h, [src, #0] + sub dst, dst, #32 /* Pre-bias. */ + ldp QB_l, QB_h, [src, #32]! /* src += 64 - Pre-bias. */ +1: + stp QA_l, QA_h, [dst, #32] + ldp QA_l, QA_h, [src, #32] + stp QB_l, QB_h, [dst, #64]! + ldp QB_l, QB_h, [src, #64]! + + subs count, count, #64 + b.ge 1b + + stp QA_l, QA_h, [dst, #32] + stp QB_l, QB_h, [dst, #64] + add src, src, #32 + add dst, dst, #64 + 32 + tst count, #0x3f + b.ne .Ltail63 + ret +.Lcpy_body_huge: + /* There are at least 128 bytes to copy. */ + ldp QA_l, QA_h, [src, #0] + sub dst, dst, #32 /* Pre-bias. */ + ldp QB_l, QB_h, [src, #32]! +1: + stnp QA_l, QA_h, [dst, #32] + stnp QB_l, QB_h, [dst, #64] + ldp QA_l, QA_h, [src, #32] + ldp QB_l, QB_h, [src, #64]! + add dst, dst, #64 + + subs count, count, #64 + b.ge 1b + + stnp QA_l, QA_h, [dst, #32] + stnp QB_l, QB_h, [dst, #64] + add src, src, #32 + add dst, dst, #64 + 32 + tst count, #0x3f + b.ne .Ltail63 + ret diff --git a/libc/arch-arm64/generic-neon/generic-neon.mk b/libc/arch-arm64/generic-neon/generic-neon.mk deleted file mode 100644 index 77e3861..0000000 --- a/libc/arch-arm64/generic-neon/generic-neon.mk +++ /dev/null @@ -1,13 +0,0 @@ -libc_bionic_src_files_arm64 += \ - arch-arm64/generic/bionic/memchr.S \ - arch-arm64/generic/bionic/memcmp.S \ - arch-arm64/generic/bionic/memmove.S \ - arch-arm64/generic/bionic/memset.S \ - arch-arm64/generic/bionic/stpcpy.S \ - arch-arm64/generic/bionic/strchr.S \ - arch-arm64/generic/bionic/strcmp.S \ - arch-arm64/generic/bionic/strcpy.S \ - arch-arm64/generic/bionic/strlen.S \ - arch-arm64/generic/bionic/strncmp.S \ - arch-arm64/generic/bionic/strnlen.S \ - arch-arm64/generic-neon/bionic/memcpy.S \ diff --git a/libc/arch-arm64/generic/bionic/memcpy.S b/libc/arch-arm64/generic/bionic/memcpy.S index e1b1a72..85129fe 100644 --- a/libc/arch-arm64/generic/bionic/memcpy.S +++ b/libc/arch-arm64/generic/bionic/memcpy.S @@ -1,184 +1,63 @@ -/* Copyright (c) 2012, Linaro Limited - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the Linaro nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* Assumptions: +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. * - * ARMv8-a, AArch64 - * Unaligned accesses + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ +// Prototype: void *memcpy (void *dst, const void *src, size_t count). + #include <private/bionic_asm.h> +#include <private/libc_events.h> -#define dstin x0 -#define src x1 -#define count x2 -#define tmp1 x3 -#define tmp1w w3 -#define tmp2 x4 -#define tmp2w w4 -#define tmp3 x5 -#define tmp3w w5 -#define dst x6 +ENTRY(__memcpy_chk) + cmp x2, x3 + b.hi __memcpy_chk_fail -#define A_l x7 -#define A_h x8 -#define B_l x9 -#define B_h x10 -#define C_l x11 -#define C_h x12 -#define D_l x13 -#define D_h x14 + // Fall through to memcpy... +END(__memcpy_chk) ENTRY(memcpy) - - mov dst, dstin - cmp count, #64 - b.ge .Lcpy_not_short - cmp count, #15 - b.le .Ltail15tiny - - /* Deal with small copies quickly by dropping straight into the - * exit block. */ -.Ltail63: - /* Copy up to 48 bytes of data. At this point we only need the - * bottom 6 bits of count to be accurate. */ - ands tmp1, count, #0x30 - b.eq .Ltail15 - add dst, dst, tmp1 - add src, src, tmp1 - cmp tmp1w, #0x20 - b.eq 1f - b.lt 2f - ldp A_l, A_h, [src, #-48] - stp A_l, A_h, [dst, #-48] -1: - ldp A_l, A_h, [src, #-32] - stp A_l, A_h, [dst, #-32] -2: - ldp A_l, A_h, [src, #-16] - stp A_l, A_h, [dst, #-16] - -.Ltail15: - ands count, count, #15 - beq 1f - add src, src, count - ldp A_l, A_h, [src, #-16] - add dst, dst, count - stp A_l, A_h, [dst, #-16] -1: - ret - -.Ltail15tiny: - /* Copy up to 15 bytes of data. Does not assume additional data - being copied. */ - tbz count, #3, 1f - ldr tmp1, [src], #8 - str tmp1, [dst], #8 -1: - tbz count, #2, 1f - ldr tmp1w, [src], #4 - str tmp1w, [dst], #4 -1: - tbz count, #1, 1f - ldrh tmp1w, [src], #2 - strh tmp1w, [dst], #2 -1: - tbz count, #0, 1f - ldrb tmp1w, [src] - strb tmp1w, [dst] -1: - ret - -.Lcpy_not_short: - /* We don't much care about the alignment of DST, but we want SRC - * to be 128-bit (16 byte) aligned so that we don't cross cache line - * boundaries on both loads and stores. */ - neg tmp2, src - ands tmp2, tmp2, #15 /* Bytes to reach alignment. */ - b.eq 2f - sub count, count, tmp2 - /* Copy more data than needed; it's faster than jumping - * around copying sub-Quadword quantities. We know that - * it can't overrun. */ - ldp A_l, A_h, [src] - add src, src, tmp2 - stp A_l, A_h, [dst] - add dst, dst, tmp2 - /* There may be less than 63 bytes to go now. */ - cmp count, #63 - b.le .Ltail63 -2: - subs count, count, #128 - b.ge .Lcpy_body_large - /* Less than 128 bytes to copy, so handle 64 here and then jump - * to the tail. */ - ldp A_l, A_h, [src] - ldp B_l, B_h, [src, #16] - ldp C_l, C_h, [src, #32] - ldp D_l, D_h, [src, #48] - stp A_l, A_h, [dst] - stp B_l, B_h, [dst, #16] - stp C_l, C_h, [dst, #32] - stp D_l, D_h, [dst, #48] - tst count, #0x3f - add src, src, #64 - add dst, dst, #64 - b.ne .Ltail63 - ret - - /* Critical loop. Start at a new cache line boundary. Assuming - * 64 bytes per line this ensures the entire loop is in one line. */ - .p2align 6 -.Lcpy_body_large: - /* There are at least 128 bytes to copy. */ - ldp A_l, A_h, [src, #0] - sub dst, dst, #16 /* Pre-bias. */ - ldp B_l, B_h, [src, #16] - ldp C_l, C_h, [src, #32] - ldp D_l, D_h, [src, #48]! /* src += 64 - Pre-bias. */ -1: - stp A_l, A_h, [dst, #16] - ldp A_l, A_h, [src, #16] - stp B_l, B_h, [dst, #32] - ldp B_l, B_h, [src, #32] - stp C_l, C_h, [dst, #48] - ldp C_l, C_h, [src, #48] - stp D_l, D_h, [dst, #64]! - ldp D_l, D_h, [src, #64]! - subs count, count, #64 - b.ge 1b - stp A_l, A_h, [dst, #16] - stp B_l, B_h, [dst, #32] - stp C_l, C_h, [dst, #48] - stp D_l, D_h, [dst, #64] - add src, src, #16 - add dst, dst, #64 + 16 - tst count, #0x3f - b.ne .Ltail63 - ret + #include "memcpy_base.S" END(memcpy) + +ENTRY_PRIVATE(__memcpy_chk_fail) + // Preserve for accurate backtrace. + stp x29, x30, [sp, -16]! + .cfi_def_cfa_offset 16 + .cfi_rel_offset x29, 0 + .cfi_rel_offset x30, 8 + + adrp x0, error_string + add x0, x0, :lo12:error_string + ldr x1, error_code + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +END(__memcpy_chk_fail) + + .data + .align 2 +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm64/generic-neon/bionic/memcpy.S b/libc/arch-arm64/generic/bionic/memcpy_base.S index 320f748..c5d42ce 100644 --- a/libc/arch-arm64/generic-neon/bionic/memcpy.S +++ b/libc/arch-arm64/generic/bionic/memcpy_base.S @@ -32,8 +32,6 @@ * */ -#include <private/bionic_asm.h> - #define dstin x0 #define src x1 #define count x2 @@ -54,13 +52,6 @@ #define D_l x13 #define D_h x14 -#define QA_l q0 -#define QA_h q1 -#define QB_l q2 -#define QB_h q3 - -ENTRY(memcpy) - mov dst, dstin cmp count, #64 b.ge .Lcpy_not_short @@ -142,10 +133,14 @@ ENTRY(memcpy) b.ge .Lcpy_body_large /* Less than 128 bytes to copy, so handle 64 here and then jump * to the tail. */ - ldp QA_l, QA_h, [src] - ldp QB_l, QB_h, [src, #32] - stp QA_l, QA_h, [dst] - stp QB_l, QB_h, [dst, #32] + ldp A_l, A_h, [src] + ldp B_l, B_h, [src, #16] + ldp C_l, C_h, [src, #32] + ldp D_l, D_h, [src, #48] + stp A_l, A_h, [dst] + stp B_l, B_h, [dst, #16] + stp C_l, C_h, [dst, #32] + stp D_l, D_h, [dst, #48] tst count, #0x3f add src, src, #64 add dst, dst, #64 @@ -157,23 +152,28 @@ ENTRY(memcpy) .p2align 6 .Lcpy_body_large: /* There are at least 128 bytes to copy. */ - ldp QA_l, QA_h, [src, #0] - sub dst, dst, #32 /* Pre-bias. */ - ldp QB_l, QB_h, [src, #32]! /* src += 64 - Pre-bias. */ + ldp A_l, A_h, [src, #0] + sub dst, dst, #16 /* Pre-bias. */ + ldp B_l, B_h, [src, #16] + ldp C_l, C_h, [src, #32] + ldp D_l, D_h, [src, #48]! /* src += 64 - Pre-bias. */ 1: - stp QA_l, QA_h, [dst, #32] - ldp QA_l, QA_h, [src, #32] - stp QB_l, QB_h, [dst, #64]! - ldp QB_l, QB_h, [src, #64]! - + stp A_l, A_h, [dst, #16] + ldp A_l, A_h, [src, #16] + stp B_l, B_h, [dst, #32] + ldp B_l, B_h, [src, #32] + stp C_l, C_h, [dst, #48] + ldp C_l, C_h, [src, #48] + stp D_l, D_h, [dst, #64]! + ldp D_l, D_h, [src, #64]! subs count, count, #64 b.ge 1b - - stp QA_l, QA_h, [dst, #32] - stp QB_l, QB_h, [dst, #64] - add src, src, #32 - add dst, dst, #64 + 32 + stp A_l, A_h, [dst, #16] + stp B_l, B_h, [dst, #32] + stp C_l, C_h, [dst, #48] + stp D_l, D_h, [dst, #64] + add src, src, #16 + add dst, dst, #64 + 16 tst count, #0x3f b.ne .Ltail63 ret -END(memcpy) diff --git a/libc/arch-arm64/include/machine/elf_machdep.h b/libc/arch-arm64/include/machine/elf_machdep.h index 2bf8189..6eab313 100644 --- a/libc/arch-arm64/include/machine/elf_machdep.h +++ b/libc/arch-arm64/include/machine/elf_machdep.h @@ -99,6 +99,7 @@ #define R_AARCH64_RELATIVE 1027 /* Adjust by program base. */ #define R_AARCH64_TLS_TPREL64 1030 #define R_AARCH64_TLS_DTPREL32 1031 +#define R_AARCH64_IRELATIVE 1032 #define R_TYPE(name) __CONCAT(R_AARCH64_,name) |