public inbox for [email protected]
 help / color / mirror / Atom feed
From: "Paul E. McKenney" <[email protected]>
To: [email protected]
Cc: [email protected], [email protected], [email protected],
	"Ammar Faizi" <[email protected]>,
	"David Laight" <[email protected]>,
	"Willy Tarreau" <[email protected]>,
	"Thomas Weißschuh" <[email protected]>
Subject: [PATCH nolibc 06/19] tools/nolibc: x86-64: Use `rep movsb` for `memcpy()` and `memmove()`
Date: Thu, 12 Oct 2023 12:32:20 -0700	[thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <b34ce3cf-3fcc-4eb0-a658-229c197455ef@paulmck-laptop>

From: Ammar Faizi <[email protected]>

Simplify memcpy() and memmove() on the x86-64 arch.

The x86-64 arch has a 'rep movsb' instruction, which can perform
memcpy() using only a single instruction, given:

    %rdi = destination
    %rsi = source
    %rcx = length

Additionally, it can also handle the overlapping case by setting DF=1
(backward copy), which can be used as the memmove() implementation.

Before this patch:
```
  00000000000010ab <memmove>:
    10ab: 48 89 f8              mov    %rdi,%rax
    10ae: 31 c9                 xor    %ecx,%ecx
    10b0: 48 39 f7              cmp    %rsi,%rdi
    10b3: 48 83 d1 ff           adc    $0xffffffffffffffff,%rcx
    10b7: 48 85 d2              test   %rdx,%rdx
    10ba: 74 25                 je     10e1 <memmove+0x36>
    10bc: 48 83 c9 01           or     $0x1,%rcx
    10c0: 48 39 f0              cmp    %rsi,%rax
    10c3: 48 c7 c7 ff ff ff ff  mov    $0xffffffffffffffff,%rdi
    10ca: 48 0f 43 fa           cmovae %rdx,%rdi
    10ce: 48 01 cf              add    %rcx,%rdi
    10d1: 44 8a 04 3e           mov    (%rsi,%rdi,1),%r8b
    10d5: 44 88 04 38           mov    %r8b,(%rax,%rdi,1)
    10d9: 48 01 cf              add    %rcx,%rdi
    10dc: 48 ff ca              dec    %rdx
    10df: 75 f0                 jne    10d1 <memmove+0x26>
    10e1: c3                    ret

  00000000000010e2 <memcpy>:
    10e2: 48 89 f8              mov    %rdi,%rax
    10e5: 48 85 d2              test   %rdx,%rdx
    10e8: 74 12                 je     10fc <memcpy+0x1a>
    10ea: 31 c9                 xor    %ecx,%ecx
    10ec: 40 8a 3c 0e           mov    (%rsi,%rcx,1),%dil
    10f0: 40 88 3c 08           mov    %dil,(%rax,%rcx,1)
    10f4: 48 ff c1              inc    %rcx
    10f7: 48 39 ca              cmp    %rcx,%rdx
    10fa: 75 f0                 jne    10ec <memcpy+0xa>
    10fc: c3                    ret
```

After this patch:
```
  // memmove is an alias for memcpy
  000000000040133b <memcpy>:
    40133b: 48 89 d1              mov    %rdx,%rcx
    40133e: 48 89 f8              mov    %rdi,%rax
    401341: 48 89 fa              mov    %rdi,%rdx
    401344: 48 29 f2              sub    %rsi,%rdx
    401347: 48 39 ca              cmp    %rcx,%rdx
    40134a: 72 03                 jb     40134f <memcpy+0x14>
    40134c: f3 a4                 rep movsb %ds:(%rsi),%es:(%rdi)
    40134e: c3                    ret
    40134f: 48 8d 7c 0f ff        lea    -0x1(%rdi,%rcx,1),%rdi
    401354: 48 8d 74 0e ff        lea    -0x1(%rsi,%rcx,1),%rsi
    401359: fd                    std
    40135a: f3 a4                 rep movsb %ds:(%rsi),%es:(%rdi)
    40135c: fc                    cld
    40135d: c3                    ret
```

v3:
  - Make memmove as an alias for memcpy (Willy).
  - Make the forward copy the likely case (Alviro).

v2:
  - Fix the broken memmove implementation (David).

Link: https://lore.kernel.org/lkml/[email protected]
Link: https://lore.kernel.org/lkml/[email protected]
Suggested-by: David Laight <[email protected]>
Signed-off-by: Ammar Faizi <[email protected]>
Signed-off-by: Willy Tarreau <[email protected]>
Signed-off-by: Thomas Weißschuh <[email protected]>
---
 tools/include/nolibc/arch-x86_64.h | 29 +++++++++++++++++++++++++++++
 tools/include/nolibc/string.h      |  4 ++++
 2 files changed, 33 insertions(+)

diff --git a/tools/include/nolibc/arch-x86_64.h b/tools/include/nolibc/arch-x86_64.h
index e5ccb926c903..aece7d895153 100644
--- a/tools/include/nolibc/arch-x86_64.h
+++ b/tools/include/nolibc/arch-x86_64.h
@@ -173,4 +173,33 @@ void __attribute__((weak, noreturn, optimize("Os", "omit-frame-pointer"))) __no_
 	__builtin_unreachable();
 }
 
+#define NOLIBC_ARCH_HAS_MEMMOVE
+void *memmove(void *dst, const void *src, size_t len);
+
+#define NOLIBC_ARCH_HAS_MEMCPY
+void *memcpy(void *dst, const void *src, size_t len);
+
+__asm__ (
+".section .text.nolibc_memmove_memcpy\n"
+".weak memmove\n"
+".weak memcpy\n"
+"memmove:\n"
+"memcpy:\n"
+	"movq %rdx, %rcx\n\t"
+	"movq %rdi, %rax\n\t"
+	"movq %rdi, %rdx\n\t"
+	"subq %rsi, %rdx\n\t"
+	"cmpq %rcx, %rdx\n\t"
+	"jb   .Lbackward_copy\n\t"
+	"rep movsb\n\t"
+	"retq\n"
+".Lbackward_copy:"
+	"leaq -1(%rdi, %rcx, 1), %rdi\n\t"
+	"leaq -1(%rsi, %rcx, 1), %rsi\n\t"
+	"std\n\t"
+	"rep movsb\n\t"
+	"cld\n\t"
+	"retq\n"
+);
+
 #endif /* _NOLIBC_ARCH_X86_64_H */
diff --git a/tools/include/nolibc/string.h b/tools/include/nolibc/string.h
index 0c2e06c7c477..6eca267ec6fa 100644
--- a/tools/include/nolibc/string.h
+++ b/tools/include/nolibc/string.h
@@ -49,6 +49,7 @@ void *_nolibc_memcpy_down(void *dst, const void *src, size_t len)
 	return dst;
 }
 
+#ifndef NOLIBC_ARCH_HAS_MEMMOVE
 /* might be ignored by the compiler without -ffreestanding, then found as
  * missing.
  */
@@ -72,13 +73,16 @@ void *memmove(void *dst, const void *src, size_t len)
 	}
 	return dst;
 }
+#endif /* #ifndef NOLIBC_ARCH_HAS_MEMMOVE */
 
+#ifndef NOLIBC_ARCH_HAS_MEMCPY
 /* must be exported, as it's used by libgcc on ARM */
 __attribute__((weak,unused,section(".text.nolibc_memcpy")))
 void *memcpy(void *dst, const void *src, size_t len)
 {
 	return _nolibc_memcpy_up(dst, src, len);
 }
+#endif /* #ifndef NOLIBC_ARCH_HAS_MEMCPY */
 
 /* might be ignored by the compiler without -ffreestanding, then found as
  * missing.
-- 
2.40.1


  parent reply	other threads:[~2023-10-12 19:32 UTC|newest]

Thread overview: 20+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-10-12 19:32 [PATCH nolibc 0/19] Updates to nolibc for v6.7 (and three for v6.6) Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 01/19] tools/nolibc: i386: Fix a stack misalign bug on _start Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 02/19] MAINTAINERS: nolibc: update tree location Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 03/19] tools/nolibc: mark start_c as weak Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 04/19] tools/nolibc: add stdarg.h header Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 05/19] selftests/nolibc: use -nostdinc for nolibc-test Paul E. McKenney
2023-10-12 19:32 ` Paul E. McKenney [this message]
2023-10-12 19:32 ` [PATCH nolibc 07/19] tools/nolibc: x86-64: Use `rep stosb` for `memset()` Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 08/19] tools/nolibc: string: Remove the `_nolibc_memcpy_down()` function Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 09/19] tools/nolibc: string: Remove the `_nolibc_memcpy_up()` function Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 10/19] selftests/nolibc: libc-test: avoid -Wstringop-overflow warnings Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 11/19] selftests/nolibc: don't embed initramfs into kernel image Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 12/19] selftests/nolibc: allow building i386 with multiarch compiler Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 13/19] tools/nolibc: avoid unused parameter warnings for ENOSYS fallbacks Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 14/19] tools/nolibc: don't define new syscall number Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 15/19] tools/nolibc: automatically detect necessity to use pselect6 Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 16/19] tools/nolibc: drop test for getauxval(AT_PAGESZ) Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 17/19] tools/nolibc: add support for constructors and destructors Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 18/19] selftests/nolibc: use qemu-system-ppc64 for ppc64le Paul E. McKenney
2023-10-12 19:32 ` [PATCH nolibc 19/19] selftests/nolibc: add tests for multi-object linkage Paul E. McKenney

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox