Skip to content

Commit ed15efe

Browse files
tlivelytlively
authored
[Parser] Templatize lexing of integers (#6272)
Have a single implementation for lexing each of unsigned, signed, and uninterpreted integers, each generic over the bit width of the integer. This reduces duplication in the existing code and it will make it much easier to support lexing more 8- and 16-bit integers.
1 parent 845e070 commit ed15efe

File tree

5 files changed

+216
-274
lines changed

5 files changed

+216
-274
lines changed

‎src/parser/input-impl.h‎

Lines changed: 16 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ inline std::optional<uint64_t> ParseInput::takeOffset() {
100100
if (subLexer == subLexer.end()) {
101101
return {};
102102
}
103-
if (auto o = subLexer->getU64()) {
103+
if (auto o = subLexer->getU<uint64_t>()) {
104104
++subLexer;
105105
if (subLexer == subLexer.end()) {
106106
++lexer;
@@ -122,7 +122,7 @@ inline std::optional<uint32_t> ParseInput::takeAlign() {
122122
if (subLexer == subLexer.end()) {
123123
return {};
124124
}
125-
if (auto a = subLexer->getU32()) {
125+
if (auto a = subLexer->getU<uint32_t>()) {
126126
++subLexer;
127127
if (subLexer == subLexer.end()) {
128128
++lexer;
@@ -134,77 +134,43 @@ inline std::optional<uint32_t> ParseInput::takeAlign() {
134134
return {};
135135
}
136136

137-
inline std::optional<uint64_t> ParseInput::takeU64() {
137+
template<typename T> inline std::optional<T> ParseInput::takeU() {
138138
if (auto t = peek()) {
139-
if (auto n = t->getU64()) {
139+
if (auto n = t->getU<T>()) {
140140
++lexer;
141141
return n;
142142
}
143143
}
144144
return std::nullopt;
145145
}
146146

147-
inline std::optional<int64_t> ParseInput::takeS64() {
147+
template<typename T> inline std::optional<T> ParseInput::takeI() {
148148
if (auto t = peek()) {
149-
if (auto n = t->getS64()) {
149+
if (auto n = t->getI<T>()) {
150150
++lexer;
151151
return n;
152152
}
153153
}
154-
return {};
154+
return std::nullopt;
155155
}
156156

157-
inline std::optional<int64_t> ParseInput::takeI64() {
158-
if (auto t = peek()) {
159-
if (auto n = t->getI64()) {
160-
++lexer;
161-
return n;
162-
}
163-
}
164-
return {};
157+
inline std::optional<uint64_t> ParseInput::takeU64() {
158+
return takeU<uint64_t>();
165159
}
166160

167-
inline std::optional<uint32_t> ParseInput::takeU32() {
168-
if (auto t = peek()) {
169-
if (auto n = t->getU32()) {
170-
++lexer;
171-
return n;
172-
}
173-
}
174-
return std::nullopt;
161+
inline std::optional<uint64_t> ParseInput::takeI64() {
162+
return takeI<uint64_t>();
175163
}
176164

177-
inline std::optional<int32_t> ParseInput::takeS32() {
178-
if (auto t = peek()) {
179-
if (auto n = t->getS32()) {
180-
++lexer;
181-
return n;
182-
}
183-
}
184-
return {};
165+
inline std::optional<uint32_t> ParseInput::takeU32() {
166+
return takeU<uint64_t>();
185167
}
186168

187-
inline std::optional<int32_t> ParseInput::takeI32() {
188-
if (auto t = peek()) {
189-
if (auto n = t->getI32()) {
190-
++lexer;
191-
return n;
192-
}
193-
}
194-
return {};
169+
inline std::optional<uint32_t> ParseInput::takeI32() {
170+
return takeI<uint32_t>();
195171
}
196172

197-
inline std::optional<uint8_t> ParseInput::takeU8() {
198-
if (auto t = peek()) {
199-
if (auto n = t->getU32()) {
200-
if (n <= std::numeric_limits<uint8_t>::max()) {
201-
++lexer;
202-
return uint8_t(*n);
203-
}
204-
}
205-
}
206-
return {};
207-
}
173+
inline std::optional<uint8_t> ParseInput::takeU8() { return takeU<uint8_t>(); }
208174

209175
inline std::optional<double> ParseInput::takeF64() {
210176
if (auto t = peek()) {

‎src/parser/input.h‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,9 @@ struct ParseInput {
5151
std::optional<uint64_t> takeOffset();
5252
std::optional<uint32_t> takeAlign();
5353
std::optional<uint64_t> takeU64();
54-
std::optional<int64_t> takeS64();
55-
std::optional<int64_t> takeI64();
54+
std::optional<uint64_t> takeI64();
5655
std::optional<uint32_t> takeU32();
57-
std::optional<int32_t> takeS32();
58-
std::optional<int32_t> takeI32();
56+
std::optional<uint32_t> takeI32();
5957
std::optional<uint8_t> takeU8();
6058
std::optional<double> takeF64();
6159
std::optional<float> takeF32();
@@ -67,6 +65,11 @@ struct ParseInput {
6765
Index getPos();
6866
[[nodiscard]] Err err(Index pos, std::string reason);
6967
[[nodiscard]] Err err(std::string reason) { return err(getPos(), reason); }
68+
69+
private:
70+
template<typename T> std::optional<T> takeU();
71+
template<typename T> std::optional<T> takeS();
72+
template<typename T> std::optional<T> takeI();
7073
};
7174

7275
#include "input-impl.h"

‎src/parser/lexer.cpp‎

Lines changed: 23 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -767,77 +767,52 @@ std::optional<LexResult> keyword(std::string_view in) {
767767

768768
} // anonymous namespace
769769

770-
std::optional<uint64_t> Token::getU64() const {
770+
template<typename T> std::optional<T> Token::getU() const {
771+
static_assert(std::is_integral_v<T> && std::is_unsigned_v<T>);
771772
if (auto* tok = std::get_if<IntTok>(&data)) {
772-
if (tok->sign == NoSign) {
773-
return tok->n;
774-
}
775-
}
776-
return {};
777-
}
778-
779-
std::optional<int64_t> Token::getS64() const {
780-
if (auto* tok = std::get_if<IntTok>(&data)) {
781-
if (tok->sign == Neg) {
782-
if (uint64_t(INT64_MIN) <= tok->n || tok->n == 0) {
783-
return int64_t(tok->n);
784-
}
785-
// TODO: Add error production for signed underflow.
786-
} else {
787-
if (tok->n <= uint64_t(INT64_MAX)) {
788-
return int64_t(tok->n);
789-
}
790-
// TODO: Add error production for signed overflow.
791-
}
792-
}
793-
return {};
794-
}
795-
796-
std::optional<uint64_t> Token::getI64() const {
797-
if (auto n = getU64()) {
798-
return *n;
799-
}
800-
if (auto n = getS64()) {
801-
return *n;
802-
}
803-
return {};
804-
}
805-
806-
std::optional<uint32_t> Token::getU32() const {
807-
if (auto* tok = std::get_if<IntTok>(&data)) {
808-
if (tok->sign == NoSign && tok->n <= UINT32_MAX) {
809-
return int32_t(tok->n);
773+
if (tok->sign == NoSign && tok->n <= std::numeric_limits<T>::max()) {
774+
return T(tok->n);
810775
}
811776
// TODO: Add error production for unsigned overflow.
812777
}
813778
return {};
814779
}
815780

816-
std::optional<int32_t> Token::getS32() const {
781+
template<typename T> std::optional<T> Token::getS() const {
782+
static_assert(std::is_integral_v<T> && std::is_signed_v<T>);
817783
if (auto* tok = std::get_if<IntTok>(&data)) {
818784
if (tok->sign == Neg) {
819-
if (uint64_t(INT32_MIN) <= tok->n || tok->n == 0) {
820-
return int32_t(tok->n);
785+
if (uint64_t(std::numeric_limits<T>::min()) <= tok->n || tok->n == 0) {
786+
return T(tok->n);
821787
}
822788
} else {
823-
if (tok->n <= uint64_t(INT32_MAX)) {
824-
return int32_t(tok->n);
789+
if (tok->n <= uint64_t(std::numeric_limits<T>::max())) {
790+
return T(tok->n);
825791
}
826792
}
827793
}
828794
return {};
829795
}
830796

831-
std::optional<uint32_t> Token::getI32() const {
832-
if (auto n = getU32()) {
797+
template<typename T> std::optional<T> Token::getI() const {
798+
static_assert(std::is_integral_v<T> && std::is_unsigned_v<T>);
799+
if (auto n = getU<T>()) {
833800
return *n;
834801
}
835-
if (auto n = getS32()) {
836-
return uint32_t(*n);
802+
if (auto n = getS<std::make_signed_t<T>>()) {
803+
return T(*n);
837804
}
838805
return {};
839806
}
840807

808+
template std::optional<uint64_t> Token::getU<uint64_t>() const;
809+
template std::optional<int64_t> Token::getS<int64_t>() const;
810+
template std::optional<uint64_t> Token::getI<uint64_t>() const;
811+
template std::optional<uint32_t> Token::getU<uint32_t>() const;
812+
template std::optional<int32_t> Token::getS<int32_t>() const;
813+
template std::optional<uint32_t> Token::getI<uint32_t>() const;
814+
template std::optional<uint8_t> Token::getU<uint8_t>() const;
815+
841816
std::optional<double> Token::getF64() const {
842817
constexpr int signif = 52;
843818
constexpr uint64_t payloadMask = (1ull << signif) - 1;

‎src/parser/lexer.h‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,10 @@ struct Token {
125125
}
126126
return {};
127127
}
128-
std::optional<uint64_t> getU64() const;
129-
std::optional<int64_t> getS64() const;
130-
std::optional<uint64_t> getI64() const;
131-
std::optional<uint32_t> getU32() const;
132-
std::optional<int32_t> getS32() const;
133-
std::optional<uint32_t> getI32() const;
128+
129+
template<typename T> std::optional<T> getU() const;
130+
template<typename T> std::optional<T> getS() const;
131+
template<typename T> std::optional<T> getI() const;
134132
std::optional<double> getF64() const;
135133
std::optional<float> getF32() const;
136134
std::optional<std::string_view> getString() const;

0 commit comments

Comments
 (0)