Skip to content

Commit 83f3920

Browse files
gchateletgchatelet
committed
[libc][NFC] Split memset implementations per platform
This is a follow up on D154800 and D154770 to make the code structure more principled and avoid too many nested #ifdef/#endif. Reviewed By: courbet Differential Revision: https://reviews.llvm.org/D155174
1 parent 3cd3f11 commit 83f3920

File tree

6 files changed

+183
-109
lines changed

6 files changed

+183
-109
lines changed

‎libc/src/string/memory_utils/CMakeLists.txt

+3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ add_header_library(
55
aarch64/bcmp_implementations.h
66
aarch64/memcmp_implementations.h
77
aarch64/memcpy_implementations.h
8+
aarch64/memset_implementations.h
89
bcmp_implementations.h
910
bzero_implementations.h
1011
generic/aligned_access.h
@@ -19,10 +20,12 @@ add_header_library(
1920
op_x86.h
2021
riscv/bcmp_implementations.h
2122
riscv/memcpy_implementations.h
23+
riscv/memset_implementations.h
2224
utils.h
2325
x86_64/bcmp_implementations.h
2426
x86_64/memcmp_implementations.h
2527
x86_64/memcpy_implementations.h
28+
x86_64/memset_implementations.h
2629
DEPS
2730
libc.src.__support.common
2831
libc.src.__support.CPP.bit

‎libc/src/string/memory_utils/aarch64/memset_implementations.h

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
//===-- Memset implementation for aarch64 -----------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H
9+
#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H
10+
11+
#include "src/__support/macros/attributes.h" // LIBC_INLINE
12+
#include "src/string/memory_utils/op_aarch64.h"
13+
#include "src/string/memory_utils/op_generic.h"
14+
#include "src/string/memory_utils/utils.h" // Ptr, CPtr
15+
16+
#include <stddef.h> // size_t
17+
18+
namespace __llvm_libc {
19+
20+
[[maybe_unused]] LIBC_INLINE static void
21+
inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
22+
static_assert(aarch64::kNeon, "aarch64 supports vector types");
23+
using uint128_t = generic_v128;
24+
using uint256_t = generic_v256;
25+
using uint512_t = generic_v512;
26+
if (count == 0)
27+
return;
28+
if (count <= 3) {
29+
generic::Memset<uint8_t>::block(dst, value);
30+
if (count > 1)
31+
generic::Memset<uint16_t>::tail(dst, value, count);
32+
return;
33+
}
34+
if (count <= 8)
35+
return generic::Memset<uint32_t>::head_tail(dst, value, count);
36+
if (count <= 16)
37+
return generic::Memset<uint64_t>::head_tail(dst, value, count);
38+
if (count <= 32)
39+
return generic::Memset<uint128_t>::head_tail(dst, value, count);
40+
if (count <= (32 + 64)) {
41+
generic::Memset<uint256_t>::block(dst, value);
42+
if (count <= 64)
43+
return generic::Memset<uint256_t>::tail(dst, value, count);
44+
generic::Memset<uint256_t>::block(dst + 32, value);
45+
generic::Memset<uint256_t>::tail(dst, value, count);
46+
return;
47+
}
48+
if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
49+
generic::Memset<uint512_t>::block(dst, 0);
50+
align_to_next_boundary<64>(dst, count);
51+
return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
52+
} else {
53+
generic::Memset<uint128_t>::block(dst, value);
54+
align_to_next_boundary<16>(dst, count);
55+
return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
56+
}
57+
}
58+
59+
} // namespace __llvm_libc
60+
61+
#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMSET_IMPLEMENTATIONS_H

‎libc/src/string/memory_utils/memset_implementations.h

+18-109
Original file line numberDiff line numberDiff line change
@@ -9,126 +9,35 @@
99
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
1010
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H
1111

12-
#include "src/__support/common.h"
13-
#include "src/__support/macros/optimization.h"
14-
#include "src/__support/macros/properties/architectures.h"
15-
#include "src/string/memory_utils/generic/aligned_access.h"
16-
#include "src/string/memory_utils/generic/byte_per_byte.h"
17-
#include "src/string/memory_utils/op_aarch64.h"
18-
#include "src/string/memory_utils/op_builtin.h"
19-
#include "src/string/memory_utils/op_generic.h"
20-
#include "src/string/memory_utils/op_x86.h"
21-
#include "src/string/memory_utils/utils.h"
12+
#include "src/__support/macros/config.h" // LIBC_INLINE
13+
#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_
14+
#include "src/string/memory_utils/utils.h" // Ptr, CPtr
2215

2316
#include <stddef.h> // size_t
2417

25-
namespace __llvm_libc {
26-
27-
#if defined(LIBC_TARGET_ARCH_IS_X86)
28-
[[maybe_unused]] LIBC_INLINE static void
29-
inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
30-
#if defined(__AVX512F__)
31-
using uint128_t = generic_v128;
32-
using uint256_t = generic_v256;
33-
using uint512_t = generic_v512;
34-
#elif defined(__AVX__)
35-
using uint128_t = generic_v128;
36-
using uint256_t = generic_v256;
37-
using uint512_t = cpp::array<generic_v256, 2>;
38-
#elif defined(__SSE2__)
39-
using uint128_t = generic_v128;
40-
using uint256_t = cpp::array<generic_v128, 2>;
41-
using uint512_t = cpp::array<generic_v128, 4>;
42-
#else
43-
using uint128_t = cpp::array<uint64_t, 2>;
44-
using uint256_t = cpp::array<uint64_t, 4>;
45-
using uint512_t = cpp::array<uint64_t, 8>;
46-
#endif
47-
48-
if (count == 0)
49-
return;
50-
if (count == 1)
51-
return generic::Memset<uint8_t>::block(dst, value);
52-
if (count == 2)
53-
return generic::Memset<uint16_t>::block(dst, value);
54-
if (count == 3)
55-
return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
56-
if (count <= 8)
57-
return generic::Memset<uint32_t>::head_tail(dst, value, count);
58-
if (count <= 16)
59-
return generic::Memset<uint64_t>::head_tail(dst, value, count);
60-
if (count <= 32)
61-
return generic::Memset<uint128_t>::head_tail(dst, value, count);
62-
if (count <= 64)
63-
return generic::Memset<uint256_t>::head_tail(dst, value, count);
64-
if (count <= 128)
65-
return generic::Memset<uint512_t>::head_tail(dst, value, count);
66-
// Aligned loop
67-
generic::Memset<uint256_t>::block(dst, value);
68-
align_to_next_boundary<32>(dst, count);
69-
return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
70-
}
71-
#endif // defined(LIBC_TARGET_ARCH_IS_X86)
72-
73-
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
74-
[[maybe_unused]] LIBC_INLINE static void
75-
inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) {
76-
static_assert(aarch64::kNeon, "aarch64 supports vector types");
77-
using uint128_t = generic_v128;
78-
using uint256_t = generic_v256;
79-
using uint512_t = generic_v512;
80-
if (count == 0)
81-
return;
82-
if (count <= 3) {
83-
generic::Memset<uint8_t>::block(dst, value);
84-
if (count > 1)
85-
generic::Memset<uint16_t>::tail(dst, value, count);
86-
return;
87-
}
88-
if (count <= 8)
89-
return generic::Memset<uint32_t>::head_tail(dst, value, count);
90-
if (count <= 16)
91-
return generic::Memset<uint64_t>::head_tail(dst, value, count);
92-
if (count <= 32)
93-
return generic::Memset<uint128_t>::head_tail(dst, value, count);
94-
if (count <= (32 + 64)) {
95-
generic::Memset<uint256_t>::block(dst, value);
96-
if (count <= 64)
97-
return generic::Memset<uint256_t>::tail(dst, value, count);
98-
generic::Memset<uint256_t>::block(dst + 32, value);
99-
generic::Memset<uint256_t>::tail(dst, value, count);
100-
return;
101-
}
102-
if (count >= 448 && value == 0 && aarch64::neon::hasZva()) {
103-
generic::Memset<uint512_t>::block(dst, 0);
104-
align_to_next_boundary<64>(dst, count);
105-
return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count);
106-
} else {
107-
generic::Memset<uint128_t>::block(dst, value);
108-
align_to_next_boundary<16>(dst, count);
109-
return generic::Memset<uint512_t>::loop_and_tail(dst, value, count);
110-
}
111-
}
112-
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)
113-
114-
LIBC_INLINE static void inline_memset(Ptr dst, uint8_t value, size_t count) {
11518
#if defined(LIBC_TARGET_ARCH_IS_X86)
116-
return inline_memset_x86(dst, value, count);
19+
#include "src/string/memory_utils/x86_64/memset_implementations.h"
20+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_x86
11721
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
118-
return inline_memset_aarch64(dst, value, count);
119-
#elif defined(LIBC_TARGET_ARCH_IS_RISCV64)
120-
return inline_memset_aligned_access_64bit(dst, value, count);
121-
#elif defined(LIBC_TARGET_ARCH_IS_RISCV32)
122-
return inline_memset_aligned_access_32bit(dst, value, count);
22+
#include "src/string/memory_utils/aarch64/memset_implementations.h"
23+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_aarch64
24+
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
25+
#include "src/string/memory_utils/riscv/memset_implementations.h"
26+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_riscv
12327
#else
124-
return inline_memset_byte_per_byte(dst, value, count);
28+
// We may want to error instead of defaulting to suboptimal implementation.
29+
#include "src/string/memory_utils/generic/byte_per_byte.h"
30+
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_byte_per_byte
12531
#endif
126-
}
32+
33+
namespace __llvm_libc {
12734

12835
LIBC_INLINE static void inline_memset(void *dst, uint8_t value, size_t count) {
129-
inline_memset(reinterpret_cast<Ptr>(dst), value, count);
36+
LIBC_SRC_STRING_MEMORY_UTILS_MEMSET(reinterpret_cast<Ptr>(dst), value, count);
13037
}
13138

13239
} // namespace __llvm_libc
13340

41+
#undef LIBC_SRC_STRING_MEMORY_UTILS_MEMSET
42+
13443
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMSET_IMPLEMENTATIONS_H

‎libc/src/string/memory_utils/riscv/memset_implementations.h

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
//===-- Memset implementation for riscv -------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H
9+
#define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H
10+
11+
#include "src/__support/macros/attributes.h" // LIBC_INLINE
12+
#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
13+
#include "src/string/memory_utils/generic/aligned_access.h"
14+
#include "src/string/memory_utils/utils.h" // Ptr, CPtr
15+
16+
#include <stddef.h> // size_t
17+
18+
namespace __llvm_libc {
19+
20+
LIBC_INLINE static void inline_memset_riscv(Ptr dst, uint8_t value,
21+
size_t count) {
22+
#if defined(LIBC_TARGET_ARCH_IS_RISCV64)
23+
return inline_memset_aligned_access_64bit(dst, value, count);
24+
#elif defined(LIBC_TARGET_ARCH_IS_RISCV32)
25+
return inline_memset_aligned_access_32bit(dst, value, count);
26+
#else
27+
#error "Unimplemented"
28+
#endif
29+
}
30+
31+
} // namespace __llvm_libc
32+
33+
#endif // LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMSET_IMPLEMENTATIONS_H

‎libc/src/string/memory_utils/x86_64/memset_implementations.h

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
//===-- Memset implementation for x86_64 ------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H
9+
#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H
10+
11+
#include "src/__support/macros/attributes.h" // LIBC_INLINE
12+
#include "src/string/memory_utils/op_generic.h"
13+
#include "src/string/memory_utils/op_x86.h"
14+
#include "src/string/memory_utils/utils.h" // Ptr, CPtr
15+
16+
#include <stddef.h> // size_t
17+
18+
namespace __llvm_libc {
19+
20+
[[maybe_unused]] LIBC_INLINE static void
21+
inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
22+
#if defined(__AVX512F__)
23+
using uint128_t = generic_v128;
24+
using uint256_t = generic_v256;
25+
using uint512_t = generic_v512;
26+
#elif defined(__AVX__)
27+
using uint128_t = generic_v128;
28+
using uint256_t = generic_v256;
29+
using uint512_t = cpp::array<generic_v256, 2>;
30+
#elif defined(__SSE2__)
31+
using uint128_t = generic_v128;
32+
using uint256_t = cpp::array<generic_v128, 2>;
33+
using uint512_t = cpp::array<generic_v128, 4>;
34+
#else
35+
using uint128_t = cpp::array<uint64_t, 2>;
36+
using uint256_t = cpp::array<uint64_t, 4>;
37+
using uint512_t = cpp::array<uint64_t, 8>;
38+
#endif
39+
40+
if (count == 0)
41+
return;
42+
if (count == 1)
43+
return generic::Memset<uint8_t>::block(dst, value);
44+
if (count == 2)
45+
return generic::Memset<uint16_t>::block(dst, value);
46+
if (count == 3)
47+
return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
48+
if (count <= 8)
49+
return generic::Memset<uint32_t>::head_tail(dst, value, count);
50+
if (count <= 16)
51+
return generic::Memset<uint64_t>::head_tail(dst, value, count);
52+
if (count <= 32)
53+
return generic::Memset<uint128_t>::head_tail(dst, value, count);
54+
if (count <= 64)
55+
return generic::Memset<uint256_t>::head_tail(dst, value, count);
56+
if (count <= 128)
57+
return generic::Memset<uint512_t>::head_tail(dst, value, count);
58+
// Aligned loop
59+
generic::Memset<uint256_t>::block(dst, value);
60+
align_to_next_boundary<32>(dst, count);
61+
return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
62+
}
63+
} // namespace __llvm_libc
64+
65+
#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMSET_IMPLEMENTATIONS_H

‎utils/bazel/llvm-project-overlay/libc/BUILD.bazel

+3
Original file line numberDiff line numberDiff line change
@@ -2012,6 +2012,7 @@ libc_support_library(
20122012
"src/string/memory_utils/aarch64/bcmp_implementations.h",
20132013
"src/string/memory_utils/aarch64/memcmp_implementations.h",
20142014
"src/string/memory_utils/aarch64/memcpy_implementations.h",
2015+
"src/string/memory_utils/aarch64/memset_implementations.h",
20152016
"src/string/memory_utils/bcmp_implementations.h",
20162017
"src/string/memory_utils/bzero_implementations.h",
20172018
"src/string/memory_utils/generic/aligned_access.h",
@@ -2023,11 +2024,13 @@ libc_support_library(
20232024
"src/string/memory_utils/memset_implementations.h",
20242025
"src/string/memory_utils/riscv/bcmp_implementations.h",
20252026
"src/string/memory_utils/riscv/memcpy_implementations.h",
2027+
"src/string/memory_utils/riscv/memset_implementations.h",
20262028
"src/string/memory_utils/strcmp_implementations.h",
20272029
"src/string/memory_utils/strstr_implementations.h",
20282030
"src/string/memory_utils/x86_64/bcmp_implementations.h",
20292031
"src/string/memory_utils/x86_64/memcmp_implementations.h",
20302032
"src/string/memory_utils/x86_64/memcpy_implementations.h",
2033+
"src/string/memory_utils/x86_64/memset_implementations.h",
20312034
],
20322035
deps = [
20332036
":__support_common",

0 commit comments

Comments
 (0)