From 88dcc843a828150bb55af3a61e05060014f0f6b8 Mon Sep 17 00:00:00 2001 From: Justin Hammond Date: Thu, 6 Aug 2026 23:24:32 +0800 Subject: [PATCH 1/2] libc/machine/risc-v: Optimize the string and memory routines. The RISC-V machine directory optimized three functions and left the rest to the generic C library. Two of the three were leaving most of their speed on the table, and the functions that suffer most on this architecture, the ones that take two pointers and have to cope with them disagreeing about alignment, were not covered at all. The two existing assembly routines learn what they most lacked. memcpy becomes register-width aware, eight bytes a step on RV64 where it always moved four, and gains a shifting path for the case it used to give up on: when source and destination disagree about where a register boundary falls, it now reads the two aligned words straddling each output word and shifts them together, so no load and no store is ever misaligned and only the head and tail go byte by byte. Its block loop is also reshaped, from ten loads followed by ten stores into four groups of four loads and four stores. That is the same instruction count in a different order, and it is worth 31% on an aligned copy: ten loads to consecutive lines fill the outstanding-miss capacity and the store burst that follows cannot overlap the next iteration's loads. strcmp stops treating every unaligned pointer as hopeless: two pointers the same distance past a boundary are walked up to it bytewise and compared a register at a time from there, which is the common case for strings carved out of larger buffers. Only pointers that disagree about the boundary keep the byte loop, because no single aligned load serves both. Six functions the directory did not cover are added as portable word-at-a-time C, sharing one small header of the old tricks: memchr, memcmp, memmove, strcpy, strncmp and strnlen. Measured on 1.4 GHz silicon, twenty runs per configuration, MB/s with 95% confidence intervals. Three builds, named here by the option that distinguishes them: default neither option set, as most boards ship today newlib CONFIG_LIBC_NEWLIB_OPTSPEED=y riscv machine CONFIG_RISCV_STRING_FUNCTION=y, this work default newlib riscv machine memcpy aligned 412 +-1 4268 +-4 3981 +-4 memcpy mismatched 410 +-1 322 +-0 3073 +-3 memmove backward 439 +-0 439 +-0 3493 +-7 memcmp aligned 31 +-0 357 +-1 414 +-1 memcmp same offset 30 +-0 38 +-0 420 +-1 memchr 645 +-2 2489 +-30 2703 +-31 strncmp aligned 32 +-0 204 +-0 268 +-0 strncmp same offset 32 +-0 27 +-0 253 +-0 strcpy aligned 609 +-1 2110 +-14 1962 +-9 strcpy same offset 616 +-1 617 +-1 1813 +-8 The pattern is the one the architecture predicts. Where both pointers are aligned newlib is already good, and beats this by 7% on memcpy and strcpy. Where the two are merely consistent with each other, newlib tests whether either pointer is aligned rather than whether the two agree, and falls to a byte loop; these walk up to the boundary and carry on a word at a time. Correctness is not assumed: every function is exercised across source and destination alignments zero through seven, twenty one sizes from zero up, overlap in both directions for memmove, terminator placement and cap interaction for the n-bounded functions, and guard bytes around every destination. The same suite passes on RV32 under qemu, and it catches deliberately injected corruption. Assisted-by: Claude:claude-opus-5 Signed-off-by: Justin Hammond --- libs/libc/machine/risc-v/CMakeLists.txt | 28 +++ libs/libc/machine/risc-v/Kconfig | 47 +++++ libs/libc/machine/risc-v/Make.defs | 28 +++ libs/libc/machine/risc-v/arch_memchr.c | 83 +++++++++ libs/libc/machine/risc-v/arch_memcmp.c | 87 ++++++++++ libs/libc/machine/risc-v/arch_memcpy.S | 220 +++++++++++++++--------- libs/libc/machine/risc-v/arch_memmove.c | 93 ++++++++++ libs/libc/machine/risc-v/arch_strcmp.S | 37 +++- libs/libc/machine/risc-v/arch_strcpy.c | 86 +++++++++ libs/libc/machine/risc-v/arch_strncmp.c | 87 ++++++++++ libs/libc/machine/risc-v/arch_strnlen.c | 75 ++++++++ libs/libc/machine/risc-v/arch_word.h | 61 +++++++ 12 files changed, 852 insertions(+), 80 deletions(-) create mode 100644 libs/libc/machine/risc-v/arch_memchr.c create mode 100644 libs/libc/machine/risc-v/arch_memcmp.c create mode 100644 libs/libc/machine/risc-v/arch_memmove.c create mode 100644 libs/libc/machine/risc-v/arch_strcpy.c create mode 100644 libs/libc/machine/risc-v/arch_strncmp.c create mode 100644 libs/libc/machine/risc-v/arch_strnlen.c create mode 100644 libs/libc/machine/risc-v/arch_word.h diff --git a/libs/libc/machine/risc-v/CMakeLists.txt b/libs/libc/machine/risc-v/CMakeLists.txt index 93ef36ef3d647..e799f146932fd 100644 --- a/libs/libc/machine/risc-v/CMakeLists.txt +++ b/libs/libc/machine/risc-v/CMakeLists.txt @@ -32,6 +32,34 @@ if(CONFIG_RISCV_STRCMP) list(APPEND SRCS arch_strcmp.S) endif() +if(CONFIG_RISCV_MEMCHR) + list(APPEND SRCS arch_memchr.c) +endif() + +if(CONFIG_RISCV_MEMCMP) + list(APPEND SRCS arch_memcmp.c) +endif() + +if(CONFIG_RISCV_MEMMOVE) + list(APPEND SRCS arch_memmove.c) +endif() + + + +if(CONFIG_RISCV_STRCPY) + list(APPEND SRCS arch_strcpy.c) +endif() + + +if(CONFIG_RISCV_STRNCMP) + list(APPEND SRCS arch_strncmp.c) +endif() + +if(CONFIG_RISCV_STRNLEN) + list(APPEND SRCS arch_strnlen.c) +endif() + + if(CONFIG_ARCH_SETJMP_H) list(APPEND SRCS arch_setjmp.S) endif() diff --git a/libs/libc/machine/risc-v/Kconfig b/libs/libc/machine/risc-v/Kconfig index 2e1dc8ff37bec..73e774f8d9241 100644 --- a/libs/libc/machine/risc-v/Kconfig +++ b/libs/libc/machine/risc-v/Kconfig @@ -7,9 +7,15 @@ config RISCV_STRING_FUNCTION bool "Enable optimized RISC-V specific string function" default n depends on ARCH_TOOLCHAIN_GNU + select RISCV_MEMCHR + select RISCV_MEMCMP select RISCV_MEMCPY + select RISCV_MEMMOVE select RISCV_MEMSET select RISCV_STRCMP + select RISCV_STRCPY + select RISCV_STRNCMP + select RISCV_STRNLEN config RISCV_MEMCPY bool "Enable optimized memcpy() for RISC-V" @@ -34,3 +40,44 @@ config RISCV_STRCMP ---help--- Enable optimized RISC-V specific strcmp() library function +config RISCV_MEMCHR + bool "Enable optimized memchr() for RISC-V" + default n + select LIBC_ARCH_MEMCHR + ---help--- + Enable optimized RISC-V specific memchr() library function + +config RISCV_MEMCMP + bool "Enable optimized memcmp() for RISC-V" + default n + select LIBC_ARCH_MEMCMP + ---help--- + Enable optimized RISC-V specific memcmp() library function + +config RISCV_MEMMOVE + bool "Enable optimized memmove() for RISC-V" + default n + select LIBC_ARCH_MEMMOVE + ---help--- + Enable optimized RISC-V specific memmove() library function + +config RISCV_STRCPY + bool "Enable optimized strcpy() for RISC-V" + default n + select LIBC_ARCH_STRCPY + ---help--- + Enable optimized RISC-V specific strcpy() library function + +config RISCV_STRNCMP + bool "Enable optimized strncmp() for RISC-V" + default n + select LIBC_ARCH_STRNCMP + ---help--- + Enable optimized RISC-V specific strncmp() library function + +config RISCV_STRNLEN + bool "Enable optimized strnlen() for RISC-V" + default n + select LIBC_ARCH_STRNLEN + ---help--- + Enable optimized RISC-V specific strnlen() library function diff --git a/libs/libc/machine/risc-v/Make.defs b/libs/libc/machine/risc-v/Make.defs index ec76a24eed484..09576d9e493da 100644 --- a/libs/libc/machine/risc-v/Make.defs +++ b/libs/libc/machine/risc-v/Make.defs @@ -32,6 +32,34 @@ ifeq ($(CONFIG_RISCV_STRCMP),y) ASRCS += arch_strcmp.S endif +ifeq ($(CONFIG_RISCV_MEMCHR),y) +CSRCS += arch_memchr.c +endif + +ifeq ($(CONFIG_RISCV_MEMCMP),y) +CSRCS += arch_memcmp.c +endif + +ifeq ($(CONFIG_RISCV_MEMMOVE),y) +CSRCS += arch_memmove.c +endif + + + +ifeq ($(CONFIG_RISCV_STRCPY),y) +CSRCS += arch_strcpy.c +endif + + +ifeq ($(CONFIG_RISCV_STRNCMP),y) +CSRCS += arch_strncmp.c +endif + +ifeq ($(CONFIG_RISCV_STRNLEN),y) +CSRCS += arch_strnlen.c +endif + + ifeq ($(CONFIG_ARCH_SETJMP_H),y) ASRCS += arch_setjmp.S endif diff --git a/libs/libc/machine/risc-v/arch_memchr.c b/libs/libc/machine/risc-v/arch_memchr.c new file mode 100644 index 0000000000000..3c54dde74efab --- /dev/null +++ b/libs/libc/machine/risc-v/arch_memchr.c @@ -0,0 +1,83 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_memchr.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_MEMCHR + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +FAR void *memchr(FAR const void *s, int c, size_t n) +{ + FAR const unsigned char *p = s; + unsigned char uc = (unsigned char)c; + WORD_T rep = WORD_REPEAT(uc); + + if (n >= 2 * WORD_BYTES) + { + while (!WORD_ALIGNED(p)) + { + if (n == 0) + { + return NULL; + } + + if (*p == uc) + { + return (FAR void *)p; + } + + p++; + n--; + } + + while (n >= WORD_BYTES && + !WORD_HASZERO(*(FAR const WORD_T *)p ^ rep)) + { + p += WORD_BYTES; + n -= WORD_BYTES; + } + } + + while (n-- > 0) + { + if (*p == uc) + { + return (FAR void *)p; + } + + p++; + } + + return NULL; +} + +#endif /* CONFIG_LIBC_ARCH_MEMCHR */ diff --git a/libs/libc/machine/risc-v/arch_memcmp.c b/libs/libc/machine/risc-v/arch_memcmp.c new file mode 100644 index 0000000000000..273eba3e84eac --- /dev/null +++ b/libs/libc/machine/risc-v/arch_memcmp.c @@ -0,0 +1,87 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_memcmp.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_MEMCMP + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +int memcmp(FAR const void *s1, FAR const void *s2, size_t n) +{ + FAR const unsigned char *a = s1; + FAR const unsigned char *b = s2; + + /* Registers at a time when both pointers sit the same distance past a + * boundary; a byte head walks them up to it. Mismatched pointers stay + * byte by byte: a shifting compare pays for itself far later than a + * shifting copy does. + */ + + if ((((uintptr_t)a ^ (uintptr_t)b) & (WORD_BYTES - 1)) == 0 && + n >= 2 * WORD_BYTES) + { + while (!WORD_ALIGNED(a)) + { + if (*a != *b) + { + return *a - *b; + } + + a++; + b++; + n--; + } + + while (n >= WORD_BYTES && + *(FAR const WORD_T *)a == *(FAR const WORD_T *)b) + { + a += WORD_BYTES; + b += WORD_BYTES; + n -= WORD_BYTES; + } + } + + while (n-- > 0) + { + if (*a != *b) + { + return *a - *b; + } + + a++; + b++; + } + + return 0; +} + +#endif /* CONFIG_LIBC_ARCH_MEMCMP */ diff --git a/libs/libc/machine/risc-v/arch_memcpy.S b/libs/libc/machine/risc-v/arch_memcpy.S index ee4cbeaae23d9..1c8e9eb571875 100644 --- a/libs/libc/machine/risc-v/arch_memcpy.S +++ b/libs/libc/machine/risc-v/arch_memcpy.S @@ -36,110 +36,172 @@ .type ARCH_LIBCFUN(memcpy), @function .file "arch_memcpy.S" -/************************************************************************************ +/* Copy a register at a time. On RV64 that is eight bytes rather than + * four, which halves the loads and the stores for the same work; RV32 + * gets exactly the code it had. + */ + +#if __riscv_xlen == 64 +# define LREG ld +# define SREG sd +# define REGBYTES 8 +# define XLENBITS 64 +#else +# define LREG lw +# define SREG sw +# define REGBYTES 4 +# define XLENBITS 32 +#endif + +/* One turn of the unrolled loop moves sixteen registers. Below this + * many bytes the alignment work costs more than it saves. + */ + +#define BLKBYTES (16 * REGBYTES) +#define SMALLSZ (4 * REGBYTES) + +/**************************************************************************** * Name: memcpy - ************************************************************************************/ + ****************************************************************************/ .text ARCH_LIBCFUN(memcpy): .cfi_sections .debug_frame .cfi_startproc - move t6, a0 /* Preserve return value */ - - /* Defer to byte-oriented copy for small sizes */ - sltiu a3, a2, 128 - bnez a3, 4f - /* Use word-oriented copy only if low-order bits match */ - andi a3, t6, 3 - andi a4, a1, 3 - bne a3, a4, 4f - - beqz a3, 2f /* Skip if already aligned */ - /* - * Round to nearest double word-aligned address - * greater than or equal to start address + mv t6, a0 /* Preserve the return value */ + + /* Below the threshold, aligning costs more than it saves */ + + li a3, SMALLSZ + bltu a2, a3, .Lbytes + + /* Align the destination first. Stores are the side worth + * aligning: a misaligned load can be fixed up by shifting what + * we already hold, a misaligned store cannot. */ - andi a3, a1, ~3 - addi a3, a3, 4 - /* Handle initial misalignment */ - sub a4, a3, a1 + + neg a3, t6 + andi a3, a3, REGBYTES-1 + beqz a3, .Ldstaligned + sub a2, a2, a3 1: - lb a5, 0(a1) + lb a4, 0(a1) addi a1, a1, 1 - sb a5, 0(t6) + sb a4, 0(t6) addi t6, t6, 1 - bltu a1, a3, 1b - sub a2, a2, a4 /* Update count */ + addi a3, a3, -1 + bnez a3, 1b + +.Ldstaligned: + /* If the source came out aligned too, copy whole registers */ + + andi a3, a1, REGBYTES-1 + bnez a3, .Lshifted + andi a4, a2, ~(BLKBYTES-1) + beqz a4, .Lwords + add a3, a1, a4 2: - andi a4, a2, ~63 - beqz a4, 4f + LREG a4, 0(a1) + LREG a5, 1*REGBYTES(a1) + LREG a6, 2*REGBYTES(a1) + LREG a7, 3*REGBYTES(a1) + SREG a4, 0(t6) + SREG a5, 1*REGBYTES(t6) + SREG a6, 2*REGBYTES(t6) + SREG a7, 3*REGBYTES(t6) + LREG a4, 4*REGBYTES(a1) + LREG a5, 5*REGBYTES(a1) + LREG a6, 6*REGBYTES(a1) + LREG a7, 7*REGBYTES(a1) + SREG a4, 4*REGBYTES(t6) + SREG a5, 5*REGBYTES(t6) + SREG a6, 6*REGBYTES(t6) + SREG a7, 7*REGBYTES(t6) + LREG a4, 8*REGBYTES(a1) + LREG a5, 9*REGBYTES(a1) + LREG a6, 10*REGBYTES(a1) + LREG a7, 11*REGBYTES(a1) + SREG a4, 8*REGBYTES(t6) + SREG a5, 9*REGBYTES(t6) + SREG a6, 10*REGBYTES(t6) + SREG a7, 11*REGBYTES(t6) + LREG a4, 12*REGBYTES(a1) + LREG a5, 13*REGBYTES(a1) + LREG a6, 14*REGBYTES(a1) + LREG a7, 15*REGBYTES(a1) + SREG a4, 12*REGBYTES(t6) + SREG a5, 13*REGBYTES(t6) + SREG a6, 14*REGBYTES(t6) + SREG a7, 15*REGBYTES(t6) + addi a1, a1, BLKBYTES + addi t6, t6, BLKBYTES + bltu a1, a3, 2b + andi a2, a2, BLKBYTES-1 + +.Lwords: + /* Whatever whole registers are left over */ + + andi a4, a2, ~(REGBYTES-1) + beqz a4, .Lbytes add a3, a1, a4 + andi a2, a2, REGBYTES-1 3: - lw a4, 0(a1) - lw a5, 4(a1) - lw a6, 2*4(a1) - lw a7, 3*4(a1) - lw t0, 4*4(a1) - lw t1, 5*4(a1) - lw t2, 6*4(a1) - lw t3, 7*4(a1) - lw t4, 8*4(a1) - lw t5, 9*4(a1) - sw a4, 0(t6) - sw a5, 4(t6) - sw a6, 2*4(t6) - sw a7, 3*4(t6) - sw t0, 4*4(t6) - sw t1, 5*4(t6) - sw t2, 6*4(t6) - sw t3, 7*4(t6) - sw t4, 8*4(t6) - sw t5, 9*4(t6) - lw a4, 10*4(a1) - lw a5, 11*4(a1) - lw a6, 12*4(a1) - lw a7, 13*4(a1) - lw t0, 14*4(a1) - lw t1, 15*4(a1) - addi a1, a1, 16*4 - sw a4, 10*4(t6) - sw a5, 11*4(t6) - sw a6, 12*4(t6) - sw a7, 13*4(t6) - sw t0, 14*4(t6) - sw t1, 15*4(t6) - addi t6, t6, 16*4 + LREG a5, 0(a1) + addi a1, a1, REGBYTES + SREG a5, 0(t6) + addi t6, t6, REGBYTES bltu a1, a3, 3b - andi a2, a2, 63 /* Update count */ + j .Lbytes + +.Lshifted: + /* The destination is aligned and the source is not, so no load + * and no store need ever be misaligned: read the two aligned + * words straddling each output word and shift them together. + * + * a3 holds how far past a boundary the source sits. It is never + * zero here, so neither shift is ever by the register width, + * which would be undefined. + */ + slli a4, a3, 3 /* Bits to shift down */ + li a5, XLENBITS + sub a5, a5, a4 /* Bits to shift the next up */ + sub a1, a1, a3 /* Round the source down */ + li a7, REGBYTES + LREG a6, 0(a1) /* The word we are part way in */ + bltu a2, a7, .Lshifttail 4: - /* Handle trailing misalignment */ - beqz a2, 6f - add a3, a1, a2 - - /* Use word-oriented copy if co-aligned to word boundary */ - or a5, a1, t6 - or a5, a5, a3 - andi a5, a5, 3 - bnez a5, 5f -7: - lw a4, 0(a1) - addi a1, a1, 4 - sw a4, 0(t6) - addi t6, t6, 4 - bltu a1, a3, 7b - - ret + LREG t0, REGBYTES(a1) + srl t1, a6, a4 + sll t2, t0, a5 + or t1, t1, t2 + SREG t1, 0(t6) + mv a6, t0 + addi a1, a1, REGBYTES + addi t6, t6, REGBYTES + sub a2, a2, a7 + bgeu a2, a7, 4b + +.Lshifttail: + add a1, a1, a3 /* Undo the rounding down */ + +.Lbytes: + /* The leftovers, and the whole copy when it was too small to be + * worth aligning. + */ + beqz a2, .Ldone + add a3, a1, a2 5: lb a4, 0(a1) addi a1, a1, 1 sb a4, 0(t6) addi t6, t6, 1 bltu a1, a3, 5b -6: + +.Ldone: ret .cfi_endproc diff --git a/libs/libc/machine/risc-v/arch_memmove.c b/libs/libc/machine/risc-v/arch_memmove.c new file mode 100644 index 0000000000000..520e521f3d921 --- /dev/null +++ b/libs/libc/machine/risc-v/arch_memmove.c @@ -0,0 +1,93 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_memmove.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_MEMMOVE + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +FAR void *memmove(FAR void *dest, FAR const void *src, size_t count) +{ + FAR unsigned char *d = dest; + FAR const unsigned char *s = src; + + if (d == s || count == 0) + { + return dest; + } + + /* Copying forward is memcpy's job, and safe whenever the start of the + * destination is below the source: by the time a source byte could be + * overwritten it has already been read. + */ + + if (d < s) + { + return memcpy(dest, src, count); + } + + /* Backward, from the top. Registers at a time when the two ends sit + * the same distance past a boundary, bytes otherwise: the mismatched + * case is rare enough for a backward overlapping copy that it is not + * worth a shifting loop of its own. + */ + + d += count; + s += count; + + if ((((uintptr_t)d ^ (uintptr_t)s) & (WORD_BYTES - 1)) == 0 && + count >= 2 * WORD_BYTES) + { + while (!WORD_ALIGNED(d)) + { + *--d = *--s; + count--; + } + + while (count >= WORD_BYTES) + { + d -= WORD_BYTES; + s -= WORD_BYTES; + count -= WORD_BYTES; + *(FAR WORD_T *)d = *(FAR const WORD_T *)s; + } + } + + while (count-- > 0) + { + *--d = *--s; + } + + return dest; +} + +#endif /* CONFIG_LIBC_ARCH_MEMMOVE */ diff --git a/libs/libc/machine/risc-v/arch_strcmp.S b/libs/libc/machine/risc-v/arch_strcmp.S index a0bad30f602c5..3ccbcfbc47b02 100644 --- a/libs/libc/machine/risc-v/arch_strcmp.S +++ b/libs/libc/machine/risc-v/arch_strcmp.S @@ -26,11 +26,46 @@ ARCH_LIBCFUN(strcmp): .cfi_sections .debug_frame .cfi_startproc - or a4, a0, a1 li t2, -1 + + /* Two pointers the same distance past a boundary can be compared + * a register at a time once both are walked up to it. Only + * pointers that disagree about where the boundary falls need the + * byte loop, since no single aligned load serves both. Test the + * difference of the pointers, not their union. + */ + + xor a4, a0, a1 and a4, a4, SZREG-1 bnez a4, .Lmisaligned + /* Same offset: walk both up to the boundary a byte at a time, + * stopping early on a difference or a terminator. + */ + + and a4, a0, SZREG-1 + beqz a4, .Laligned +.Lhead: + lbu a2, 0(a0) + lbu a3, 0(a1) + bne a2, a3, .Lheaddiff + addi a0, a0, 1 + addi a1, a1, 1 + beqz a2, .Lheadeq + and a4, a0, SZREG-1 + bnez a4, .Lhead + j .Laligned + +.Lheaddiff: + sub a0, a2, a3 + ret + +.Lheadeq: + li a0, 0 + ret + +.Laligned: + #if SZREG == 4 li a5, 0x7f7f7f7f #else diff --git a/libs/libc/machine/risc-v/arch_strcpy.c b/libs/libc/machine/risc-v/arch_strcpy.c new file mode 100644 index 0000000000000..6cbe79dba7c28 --- /dev/null +++ b/libs/libc/machine/risc-v/arch_strcpy.c @@ -0,0 +1,86 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_strcpy.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_STRCPY + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +FAR char *strcpy(FAR char *dest, FAR const char *src) +{ + FAR const WORD_T *ws; + FAR WORD_T *wd; + FAR char *d = dest; + + /* Words only when the two pointers agree about where boundaries fall, + * since the stores must be aligned and a shifting loop is not worth it + * for typical string lengths. Note that this asks whether the two + * agree modulo the word size, not whether either is aligned: a pair + * that is offset by the same amount is walked up to the boundary and + * then copied a word at a time like any other. + */ + + if ((((uintptr_t)d ^ (uintptr_t)src) & (WORD_BYTES - 1)) == 0) + { + while (!WORD_ALIGNED(src)) + { + if ((*d++ = *src++) == '\0') + { + return dest; + } + } + + /* Cursors of the word type, so that the copy is one load and one + * store per turn. Re-casting the char pointers inside the loop + * costs a reload, since the store through one may alias the other. + */ + + wd = (FAR WORD_T *)d; + ws = (FAR const WORD_T *)src; + + while (!WORD_HASZERO(*ws)) + { + *wd++ = *ws++; + } + + d = (FAR char *)wd; + src = (FAR const char *)ws; + } + + while ((*d++ = *src++) != '\0') + { + } + + return dest; +} + +#endif /* CONFIG_LIBC_ARCH_STRCPY */ diff --git a/libs/libc/machine/risc-v/arch_strncmp.c b/libs/libc/machine/risc-v/arch_strncmp.c new file mode 100644 index 0000000000000..da1eaf6846185 --- /dev/null +++ b/libs/libc/machine/risc-v/arch_strncmp.c @@ -0,0 +1,87 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_strncmp.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_STRNCMP + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +int strncmp(FAR const char *cs, FAR const char *ct, size_t nb) +{ + FAR const unsigned char *a = (FAR const unsigned char *)cs; + FAR const unsigned char *b = (FAR const unsigned char *)ct; + + if ((((uintptr_t)a ^ (uintptr_t)b) & (WORD_BYTES - 1)) == 0 && + nb >= 2 * WORD_BYTES) + { + while (!WORD_ALIGNED(a)) + { + if (*a != *b || *a == '\0') + { + return *a - *b; + } + + a++; + b++; + nb--; + } + + /* A word is passed over only while the two agree and neither + * holds the terminator, so whatever the loop stops on can be + * settled by the byte loop below inside one word. + */ + + while (nb >= WORD_BYTES && + *(FAR const WORD_T *)a == *(FAR const WORD_T *)b && + !WORD_HASZERO(*(FAR const WORD_T *)a)) + { + a += WORD_BYTES; + b += WORD_BYTES; + nb -= WORD_BYTES; + } + } + + while (nb-- > 0) + { + if (*a != *b || *a == '\0') + { + return *a - *b; + } + + a++; + b++; + } + + return 0; +} + +#endif /* CONFIG_LIBC_ARCH_STRNCMP */ diff --git a/libs/libc/machine/risc-v/arch_strnlen.c b/libs/libc/machine/risc-v/arch_strnlen.c new file mode 100644 index 0000000000000..e784bd801237a --- /dev/null +++ b/libs/libc/machine/risc-v/arch_strnlen.c @@ -0,0 +1,75 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_strnlen.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_STRNLEN + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +size_t strnlen(FAR const char *s, size_t maxlen) +{ + FAR const char *p = s; + FAR const char *end = s + maxlen; + + /* The cap may sit anywhere, including inside the word that holds the + * terminator, so the word loop must stop a whole word short of it and + * leave the rest to the byte tail. + */ + + if (maxlen >= WORD_BYTES) + { + while (!WORD_ALIGNED(p)) + { + if (p == end || *p == '\0') + { + return p - s; + } + + p++; + } + + while ((size_t)(end - p) >= WORD_BYTES && + !WORD_HASZERO(*(FAR const WORD_T *)p)) + { + p += WORD_BYTES; + } + } + + while (p != end && *p != '\0') + { + p++; + } + + return p - s; +} + +#endif /* CONFIG_LIBC_ARCH_STRNLEN */ diff --git a/libs/libc/machine/risc-v/arch_word.h b/libs/libc/machine/risc-v/arch_word.h new file mode 100644 index 0000000000000..9cf6ce69c1755 --- /dev/null +++ b/libs/libc/machine/risc-v/arch_word.h @@ -0,0 +1,61 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_word.h + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __LIBS_LIBC_MACHINE_RISCV_ARCH_WORD_H +#define __LIBS_LIBC_MACHINE_RISCV_ARCH_WORD_H + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/* Scanning a register at a time instead of a byte at a time. + * + * With ones in every byte's low bit and in every byte's high bit, + * + * (x - ONES) & ~x & HIGHS + * + * is nonzero exactly when some byte of x is zero. Subtracting one from a + * zero byte borrows into its high bit while ~x keeps only bytes that had + * no high bit of their own, so only genuinely zero bytes survive the + * mask. Finding a particular byte is the same test applied to x XOR a + * word with that byte repeated. + * + * uintptr_t is the native register on both RV32 and RV64, so the same + * source is four bytes a step on one and eight on the other. + */ + +#define WORD_T uintptr_t +#define WORD_BYTES sizeof(WORD_T) +#define WORD_ONES ((WORD_T)-1 / 0xff) /* 0x0101..01 */ +#define WORD_HIGHS (WORD_ONES << 7) /* 0x8080..80 */ + +#define WORD_HASZERO(x) (((x) - WORD_ONES) & ~(x) & WORD_HIGHS) +#define WORD_REPEAT(c) (WORD_ONES * (unsigned char)(c)) +#define WORD_ALIGNED(p) ((((uintptr_t)(p)) & (WORD_BYTES - 1)) == 0) + +#endif /* __LIBS_LIBC_MACHINE_RISCV_ARCH_WORD_H */ From 1fc8c835fcd94445b509e9b048d317443f22ce60 Mon Sep 17 00:00:00 2001 From: Justin Hammond Date: Sat, 8 Aug 2026 00:25:18 +0800 Subject: [PATCH 2/2] libc/machine/risc-v: Add strlcpy, and unroll the two compare loops. strlcpy earns its place by the numbers: forty six call sites in one kernel image, more than strcpy, and neither the generic library nor the arm64 directory offers an optimized one. The word loop keeps a byte in hand so the terminator always fits, and the length it must return regardless of truncation is finished by strlen. Measured over twenty runs at 1818 +-9 MB/s against the generic's 466 +-1, a byte pace. The two compare loops learn to carry one branch for several words: memcmp folds four words' differences together with XOR and OR before testing, strncmp two words' differences and terminators. Whatever stops the loop is then within a few words and the byte tail settles it. The honest measurement is that this helps less than it should, 343 to 420 MB/s for memcmp and 245 to 272 for strncmp, while the emitted loop is eight loads, four XORs, three ORs and a branch per thirty two bytes, and a single-stream word loop on this core runs at 3.3 GB/s. Something about two-stream reads here deserves a profile of its own; the loops are left unrolled because they are no worse anywhere and the shape is right once that is understood. Correctness for the new function: every source and destination alignment, lengths zero to twenty three, and every cap from zero to past the end. The return is always the source length, the result is terminated whenever the cap is nonzero, at most cap minus one bytes are copied, and a cap of zero writes nothing. Verified on RV64 silicon and RV32 under qemu alongside the whole existing suite. Assisted-by: Claude:claude-opus-5 Signed-off-by: Justin Hammond --- libs/libc/machine/risc-v/CMakeLists.txt | 6 +- libs/libc/machine/risc-v/Kconfig | 8 +++ libs/libc/machine/risc-v/Make.defs | 3 + libs/libc/machine/risc-v/arch_memcmp.c | 26 ++++++++ libs/libc/machine/risc-v/arch_strlcpy.c | 88 +++++++++++++++++++++++++ libs/libc/machine/risc-v/arch_strncmp.c | 25 +++++-- 6 files changed, 146 insertions(+), 10 deletions(-) create mode 100644 libs/libc/machine/risc-v/arch_strlcpy.c diff --git a/libs/libc/machine/risc-v/CMakeLists.txt b/libs/libc/machine/risc-v/CMakeLists.txt index e799f146932fd..8aae98f27c28c 100644 --- a/libs/libc/machine/risc-v/CMakeLists.txt +++ b/libs/libc/machine/risc-v/CMakeLists.txt @@ -44,12 +44,13 @@ if(CONFIG_RISCV_MEMMOVE) list(APPEND SRCS arch_memmove.c) endif() - - if(CONFIG_RISCV_STRCPY) list(APPEND SRCS arch_strcpy.c) endif() +if(CONFIG_RISCV_STRLCPY) + list(APPEND SRCS arch_strlcpy.c) +endif() if(CONFIG_RISCV_STRNCMP) list(APPEND SRCS arch_strncmp.c) @@ -59,7 +60,6 @@ if(CONFIG_RISCV_STRNLEN) list(APPEND SRCS arch_strnlen.c) endif() - if(CONFIG_ARCH_SETJMP_H) list(APPEND SRCS arch_setjmp.S) endif() diff --git a/libs/libc/machine/risc-v/Kconfig b/libs/libc/machine/risc-v/Kconfig index 73e774f8d9241..4db37fe831b5b 100644 --- a/libs/libc/machine/risc-v/Kconfig +++ b/libs/libc/machine/risc-v/Kconfig @@ -14,6 +14,7 @@ config RISCV_STRING_FUNCTION select RISCV_MEMSET select RISCV_STRCMP select RISCV_STRCPY + select RISCV_STRLCPY select RISCV_STRNCMP select RISCV_STRNLEN @@ -81,3 +82,10 @@ config RISCV_STRNLEN select LIBC_ARCH_STRNLEN ---help--- Enable optimized RISC-V specific strnlen() library function + +config RISCV_STRLCPY + bool "Enable optimized strlcpy() for RISC-V" + default n + select LIBC_ARCH_STRLCPY + ---help--- + Enable optimized RISC-V specific strlcpy() library function diff --git a/libs/libc/machine/risc-v/Make.defs b/libs/libc/machine/risc-v/Make.defs index 09576d9e493da..cab4cda3a0790 100644 --- a/libs/libc/machine/risc-v/Make.defs +++ b/libs/libc/machine/risc-v/Make.defs @@ -50,6 +50,9 @@ ifeq ($(CONFIG_RISCV_STRCPY),y) CSRCS += arch_strcpy.c endif +ifeq ($(CONFIG_RISCV_STRLCPY),y) +CSRCS += arch_strlcpy.c +endif ifeq ($(CONFIG_RISCV_STRNCMP),y) CSRCS += arch_strncmp.c diff --git a/libs/libc/machine/risc-v/arch_memcmp.c b/libs/libc/machine/risc-v/arch_memcmp.c index 273eba3e84eac..474f0be3bab2b 100644 --- a/libs/libc/machine/risc-v/arch_memcmp.c +++ b/libs/libc/machine/risc-v/arch_memcmp.c @@ -61,6 +61,32 @@ int memcmp(FAR const void *s1, FAR const void *s2, size_t n) n--; } + /* Four words a turn, their differences OR-ed together so the + * loop carries a single branch. On a difference, fall out and + * let the byte loop find it: it is within four words of here. + */ + + while (n >= 4 * WORD_BYTES) + { + WORD_T x = (((FAR const WORD_T *)a)[0] ^ + ((FAR const WORD_T *)b)[0]) | + (((FAR const WORD_T *)a)[1] ^ + ((FAR const WORD_T *)b)[1]) | + (((FAR const WORD_T *)a)[2] ^ + ((FAR const WORD_T *)b)[2]) | + (((FAR const WORD_T *)a)[3] ^ + ((FAR const WORD_T *)b)[3]); + + if (x != 0) + { + break; + } + + a += 4 * WORD_BYTES; + b += 4 * WORD_BYTES; + n -= 4 * WORD_BYTES; + } + while (n >= WORD_BYTES && *(FAR const WORD_T *)a == *(FAR const WORD_T *)b) { diff --git a/libs/libc/machine/risc-v/arch_strlcpy.c b/libs/libc/machine/risc-v/arch_strlcpy.c new file mode 100644 index 0000000000000..f7458e43abc50 --- /dev/null +++ b/libs/libc/machine/risc-v/arch_strlcpy.c @@ -0,0 +1,88 @@ +/**************************************************************************** + * libs/libc/machine/risc-v/arch_strlcpy.c + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include "arch_word.h" +#include "libc.h" + +#ifdef CONFIG_LIBC_ARCH_STRLCPY + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +size_t strlcpy(FAR char *dst, FAR const char *src, size_t dsize) +{ + FAR const char *s = src; + FAR char *d = dst; + size_t remain = dsize; + + /* Words while the pointers agree about boundaries, there is room for a + * whole word plus the terminator, and no byte of the word is the + * terminator. The cap keeps a byte back so the terminator always + * fits. + */ + + if ((((uintptr_t)d ^ (uintptr_t)s) & (WORD_BYTES - 1)) == 0 && + remain > WORD_BYTES) + { + while (!WORD_ALIGNED(s) && remain > 1 && *s != '\0') + { + *d++ = *s++; + remain--; + } + + while (remain > WORD_BYTES && + !WORD_HASZERO(*(FAR const WORD_T *)s)) + { + *(FAR WORD_T *)d = *(FAR const WORD_T *)s; + d += WORD_BYTES; + s += WORD_BYTES; + remain -= WORD_BYTES; + } + } + + while (remain > 1 && *s != '\0') + { + *d++ = *s++; + remain--; + } + + if (remain > 0) + { + *d = '\0'; + } + + /* strlcpy returns the source length regardless of how much was copied, + * which is how the caller detects truncation. The remainder is a plain + * strlen, already the fast one. + */ + + return (s - src) + strlen(s); +} + +#endif /* CONFIG_LIBC_ARCH_STRLCPY */ diff --git a/libs/libc/machine/risc-v/arch_strncmp.c b/libs/libc/machine/risc-v/arch_strncmp.c index da1eaf6846185..be9b6e2ed3363 100644 --- a/libs/libc/machine/risc-v/arch_strncmp.c +++ b/libs/libc/machine/risc-v/arch_strncmp.c @@ -57,16 +57,27 @@ int strncmp(FAR const char *cs, FAR const char *ct, size_t nb) /* A word is passed over only while the two agree and neither * holds the terminator, so whatever the loop stops on can be - * settled by the byte loop below inside one word. + * settled by the byte loop below inside two words. Differences + * and terminators are OR-ed into one value so each pair of words + * costs a single branch. */ - while (nb >= WORD_BYTES && - *(FAR const WORD_T *)a == *(FAR const WORD_T *)b && - !WORD_HASZERO(*(FAR const WORD_T *)a)) + while (nb >= 2 * WORD_BYTES) { - a += WORD_BYTES; - b += WORD_BYTES; - nb -= WORD_BYTES; + WORD_T w0 = ((FAR const WORD_T *)a)[0]; + WORD_T w1 = ((FAR const WORD_T *)a)[1]; + WORD_T x = (w0 ^ ((FAR const WORD_T *)b)[0]) | + (w1 ^ ((FAR const WORD_T *)b)[1]) | + WORD_HASZERO(w0) | WORD_HASZERO(w1); + + if (x != 0) + { + break; + } + + a += 2 * WORD_BYTES; + b += 2 * WORD_BYTES; + nb -= 2 * WORD_BYTES; } }