Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions JSTests/stress/out-of-memory-in-globalFuncUnescape.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
//@ memoryHog!
//@ if $buildType == "release" then runDefault else skip end
//@ if $buildType == "debug" then runDefault("--maxSingleAllocationSize=1048576") else skip end

// With this limit on the size of one allocation, the 8-bit input fits and the 16-bit result does not.
var exception;
try {
unescape('\u0100'.repeat(2**30));
unescape('%u0100' + 'a'.repeat(600000));
} catch (e) {
exception = e;
}
Expand Down
6 changes: 3 additions & 3 deletions JSTests/stress/regexp-escape-oom.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
//@ memoryHog!
//@ runDefault
//@ if $buildType == "debug" then runDefault("--maxSingleAllocationSize=1048576") else skip end

// With this limit on the size of one allocation, the input fits and the result does not. Each "^" becomes "\^".
var exception;
try {
RegExp.escape('퀀'.repeat?.(2 ** 30));
RegExp.escape('퀀' + '^'.repeat?.(300000));
} catch (e) {
exception = e;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//@ memoryHog!
//@ skip if $buildType == "debug" or $addressBits <= 32
//@ runDefault

// unescape() reserves an 8-bit StringBuilder buffer of the input length, and the first %uXXXX converts
// that buffer to 16-bit. Twice the input length is more than the longest 16-bit string here. The result
// is a sixth of the input length, so this must not throw an out of memory error.
const count = 178956971;
const input = "%u1234".repeat(count);
if (input.length <= 2 ** 30)
throw new Error("input is too short: " + input.length);

const output = unescape(input);
if (output.length !== count)
throw new Error("bad length: " + output.length);
if (output.charCodeAt(0) !== 0x1234 || output.charCodeAt(count - 1) !== 0x1234)
throw new Error("bad content");
10 changes: 1 addition & 9 deletions Source/WTF/wtf/text/StringBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,6 @@

namespace WTF {

static constexpr unsigned maxCapacity = String::MaxLength;

unsigned StringBuilder::expandedCapacity(unsigned capacity, unsigned requiredCapacity)
{
static constexpr unsigned minimumCapacity = 16;
return std::max(requiredCapacity, std::max(minimumCapacity, std::min(capacity * 2, maxCapacity)));
}

void StringBuilder::didOverflow()
{
if (m_shouldCrashOnOverflow)
Expand Down Expand Up @@ -141,7 +133,7 @@ std::span<Latin1Character> StringBuilder::extendBufferForAppendingLatin1Characte
std::span<char16_t> StringBuilder::extendBufferForAppendingWithUpconvert(unsigned requiredLength)
{
if (is8Bit()) {
allocateBuffer<char16_t>(span8(), expandedCapacity(capacity(), requiredLength));
allocateBuffer<char16_t>(span8(), expandedCapacity<char16_t>(capacity(), requiredLength));
if (hasOverflowed()) [[unlikely]]
return { };
return spanConstCast<char16_t>(m_buffer->span16().subspan(std::exchange(m_length, requiredLength)));
Expand Down
2 changes: 1 addition & 1 deletion Source/WTF/wtf/text/StringBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ class StringBuilder {
WTF_EXPORT_PRIVATE bool NODELETE containsOnlyASCII() const;

private:
static unsigned expandedCapacity(unsigned capacity, unsigned requiredCapacity);
template<typename CharacterType> static unsigned expandedCapacity(unsigned capacity, unsigned requiredCapacity);

template<typename AllocationCharacterType, typename CurrentCharacterType> void allocateBuffer(std::span<const CurrentCharacterType> currentCharacters, unsigned requiredCapacity);
template<typename CharacterType> void reallocateBuffer(unsigned requiredCapacity);
Expand Down
10 changes: 9 additions & 1 deletion Source/WTF/wtf/text/StringBuilderInternals.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,14 @@

namespace WTF {

// CharacterType is the type of the buffer the returned capacity is for.
template<typename CharacterType> unsigned StringBuilder::expandedCapacity(unsigned capacity, unsigned requiredCapacity)
{
static constexpr unsigned minimumCapacity = 16;
static constexpr unsigned maxCapacity = StringImpl::maxValidLength<CharacterType>();
return std::max(requiredCapacity, std::max(minimumCapacity, std::min(capacity * 2, maxCapacity)));
}

// Allocate a new buffer, copying in currentCharacters (these may come from either m_string or m_buffer.
template<typename AllocationCharacterType, typename CurrentCharacterType> void StringBuilder::allocateBuffer(std::span<const CurrentCharacterType> currentCharactersToCopy, unsigned requiredCapacity)
{
Expand Down Expand Up @@ -85,7 +93,7 @@ template<typename CharacterType> std::span<CharacterType> StringBuilder::extendB
{
if (!requiredLength || hasOverflowed())
return { };
reallocateBuffer(expandedCapacity(capacity(), requiredLength));
reallocateBuffer(expandedCapacity<CharacterType>(capacity(), requiredLength));
if (hasOverflowed()) [[unlikely]]
return { };
return spanConstCast<CharacterType>(m_buffer->span<CharacterType>().subspan(std::exchange(m_length, requiredLength)));
Expand Down
12 changes: 10 additions & 2 deletions Source/WTF/wtf/text/StringImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ class StringImpl : private StringImplShape, public NoVirtualDestructorBase {

// Prefer using isValidLength over MaxLength when the character type is known.
template<typename> static constexpr bool isValidLength(size_t);
// The longest length isValidLength accepts. For char16_t it is less than MaxLength.
template<typename> static constexpr unsigned maxValidLength();

static constexpr unsigned MaxLength = StringImplShape::MaxLength;

Expand Down Expand Up @@ -1261,10 +1263,16 @@ template<typename T> inline size_t StringImpl::allocationSize(Checked<size_t> ta
}

template<typename CharacterType>
inline constexpr bool StringImpl::isValidLength(size_t length)
inline constexpr unsigned StringImpl::maxValidLength()
{
// In order to not overflow the unsigned length, the check for (std::numeric_limits<unsigned>::max() - sizeof(StringImpl)) is needed when sizeof(CharacterType) == 2.
constexpr size_t max = std::min(static_cast<size_t>(MaxLength), (std::numeric_limits<unsigned>::max() - sizeof(StringImpl)) / sizeof(CharacterType));
return static_cast<unsigned>(std::min(static_cast<size_t>(MaxLength), (std::numeric_limits<unsigned>::max() - sizeof(StringImpl)) / sizeof(CharacterType)));
}

template<typename CharacterType>
inline constexpr bool StringImpl::isValidLength(size_t length)
{
constexpr size_t max = maxValidLength<CharacterType>();
return length <= max;
}

Expand Down
127 changes: 127 additions & 0 deletions Tools/TestWebKitAPI/Tests/WTF/StringBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,41 @@
#include "Helpers/Test.h"
#include "Helpers/WTFTestUtilities.h"
#include <sstream>
#include <wtf/DataLog.h>
#include <wtf/text/MakeString.h>
#include <wtf/unicode/CharacterNames.h>

namespace TestWebKitAPI {

// A run of 16-bit characters that a StringBuilder makes room for and that nobody writes.
// A test can make a builder need a very long buffer with it and touch none of the memory.
struct UnwrittenCharacters {
unsigned length;
};

} // namespace TestWebKitAPI

namespace WTF {

template<> class StringTypeAdapter<TestWebKitAPI::UnwrittenCharacters> {
public:
StringTypeAdapter(TestWebKitAPI::UnwrittenCharacters characters)
: m_length { characters.length }
{
}

unsigned length() const { return m_length; }
bool is8Bit() const { return false; }
template<typename CharacterType> void writeTo(std::span<CharacterType>) const { }

private:
unsigned m_length;
};

} // namespace WTF

namespace TestWebKitAPI {

static String builderContent(const StringBuilder& builder)
{
// Not using builder.toString() or builder.toStringPreserveCapacity() because they all
Expand Down Expand Up @@ -442,6 +472,103 @@ TEST(StringBuilderTest, ShouldShrinkToFit)
EXPECT_FALSE(builder.shouldShrinkToFit());
}

// The longest 16-bit string is shorter than String::MaxLength, because the size of a StringImpl has to fit in an unsigned.
static constexpr unsigned maxLength16Bit = StringImpl::maxValidLength<char16_t>();
static_assert(maxLength16Bit < String::MaxLength);
static_assert(StringImpl::isValidLength<char16_t>(maxLength16Bit) && !StringImpl::isValidLength<char16_t>(static_cast<size_t>(maxLength16Bit) + 1));
static_assert(StringImpl::maxValidLength<Latin1Character>() == String::MaxLength);

// Twice this capacity is more than the longest 16-bit string.
static constexpr unsigned capacityAboveHalfOfMaxLength16Bit = maxLength16Bit / 2 + 1;
static_assert(capacityAboveHalfOfMaxLength16Bit * 2 > maxLength16Bit);

// The tests that call this reserve up to 6GB of address space and write to almost none of it. They return early
// on a machine that cannot reserve that much. They do not call GTEST_SKIP(), because the TestWebKitAPI listener
// reports a skipped test as a failure.
template<typename FirstCharacterType, typename SecondCharacterType>
static bool canAllocateBothBuffers(unsigned firstLength, unsigned secondLength)
{
std::span<FirstCharacterType> firstCharacters;
std::span<SecondCharacterType> secondCharacters;
auto firstBuffer = StringImpl::tryCreateUninitialized(firstLength, firstCharacters);
auto secondBuffer = StringImpl::tryCreateUninitialized(secondLength, secondCharacters);
if (firstBuffer && secondBuffer)
return true;
dataLogLn("Cannot allocate the buffers for this test, so the test did not run.");
return false;
}

TEST(StringBuilderTest, UpconvertCapacityAboveHalfOfMaxLength)
{
constexpr unsigned capacity8Bit = capacityAboveHalfOfMaxLength16Bit;
if (!canAllocateBothBuffers<Latin1Character, char16_t>(capacity8Bit, maxLength16Bit))
return;

StringBuilder builder(OverflowPolicy::RecordOverflow);
builder.reserveCapacity(capacity8Bit);
builder.append("ab"_s);
ASSERT_FALSE(builder.hasOverflowed());
EXPECT_TRUE(builder.is8Bit());
EXPECT_EQ(capacity8Bit, builder.capacity());

const char16_t nonLatin1 = 0x1234;
builder.append(nonLatin1);
ASSERT_FALSE(builder.hasOverflowed());
EXPECT_FALSE(builder.is8Bit());
EXPECT_GE(builder.capacity(), capacity8Bit);
EXPECT_LE(builder.capacity(), maxLength16Bit);
EXPECT_EQ(3U, builder.length());
EXPECT_EQ('a', static_cast<char>(builder[0]));
EXPECT_EQ('b', static_cast<char>(builder[1]));
EXPECT_EQ(nonLatin1, builder[2]);
}

TEST(StringBuilderTest, Grow16BitCapacityAboveHalfOfMaxLength)
{
constexpr unsigned capacity = capacityAboveHalfOfMaxLength16Bit;
if (!canAllocateBothBuffers<char16_t, char16_t>(capacity, maxLength16Bit))
return;

const char16_t nonLatin1 = 0x1234;
constexpr unsigned writtenLength = 16;

StringBuilder builder(OverflowPolicy::RecordOverflow);
for (unsigned i = 0; i < writtenLength; ++i)
builder.append(nonLatin1);
builder.reserveCapacity(capacity);
ASSERT_FALSE(builder.hasOverflowed());
EXPECT_FALSE(builder.is8Bit());
EXPECT_EQ(capacity, builder.capacity());

// This string shares the buffer with the builder. A builder that is not the only owner of its buffer copies
// its characters to a new buffer. A builder that is the only owner reallocates, and that can copy all 2GB.
String sharesBuffer = builder.toStringPreserveCapacity();

builder.append(UnwrittenCharacters { capacity - writtenLength + 1 });
ASSERT_FALSE(builder.hasOverflowed());
EXPECT_EQ(capacity + 1, builder.length());
EXPECT_GE(builder.capacity(), builder.length());
EXPECT_LE(builder.capacity(), maxLength16Bit);
EXPECT_EQ(nonLatin1, builder[0]);
EXPECT_EQ(nonLatin1, builder[writtenLength - 1]);
EXPECT_EQ(writtenLength, sharesBuffer.length());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
EXPECT_EQ(nonLatin1, sharesBuffer[0]);
EXPECT_EQ(nonLatin1, sharesBuffer[writtenLength - 1]);
}

TEST(StringBuilderTest, LengthAboveMaxLength16BitOverflows)
{
const char16_t nonLatin1 = 0x1234;
StringBuilder builder(OverflowPolicy::RecordOverflow);
builder.append(nonLatin1);
ASSERT_FALSE(builder.hasOverflowed());
EXPECT_FALSE(builder.is8Bit());

// The longest 16-bit string plus one character. The builder has to refuse this length, whatever capacity it picks.
builder.append(UnwrittenCharacters { maxLength16Bit });
EXPECT_TRUE(builder.hasOverflowed());
}

TEST(StringBuilderTest, ToAtomString)
{
StringBuilder builder;
Expand Down
Loading