Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 1 addition & 0 deletions Source/JavaScriptCore/Sources.txt
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ b3/B3PhiChildren.cpp
b3/B3PlaceConstantsByFrequency.cpp @no-unify
b3/B3Policy.cpp @no-unify
b3/B3Procedure.cpp
b3/B3PromoteLoopMemory.cpp @no-unify
b3/B3PureCSE.cpp
b3/B3RangeReductions.cpp @no-unify
b3/B3ReduceDoubleToFloat.cpp
Expand Down
15 changes: 15 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTEscapeAnalysis.h
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,8 @@ class EscapeAnalysis {
std::optional<ObjectID> objectDefinedBy(ValueID) const;

PointsToSet pointsTo(ValueID) const LIFETIME_BOUND;
// The allocations of the function that the value may be, where it can be nothing else. Two values with such sets that share no member are never one object.
std::optional<ObjectSet> allocationsIfNothingElse(ValueID) const LIFETIME_BOUND;
std::optional<ObjectID> singleObject(ValueID) const;
bool isExactUse(ObjectID, const Use&) const;
bool isOnlyOperandHolding(ObjectID, const Use&) const;
Expand Down Expand Up @@ -472,6 +474,19 @@ inline PointsToSet EscapeAnalysis::pointsTo(ValueID value) const
return { ObjectSet(set.objects), set.mayBeUntracked };
}

inline std::optional<ObjectSet> EscapeAnalysis::allocationsIfNothingElse(ValueID value) const
{
solveIfWaiting();
RELEASE_ASSERT(value.index() < m_setOfValue.size());
ValueSetID id = m_setOfValue[value.index()];
if (!id)
return std::nullopt;
const StoredSet& set = m_sets[id.index()];
if (set.mayBeUntracked || set.holdsEntryParameter)
return std::nullopt;
return ObjectSet(set.objects);
}

inline std::optional<ObjectID> EscapeAnalysis::singleObject(ValueID value) const
{
PointsToSet set = pointsTo(value);
Expand Down
2 changes: 1 addition & 1 deletion Source/JavaScriptCore/aot/analysis/AOTFunctionAnalyses.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ static AnalysisSet addDependentAnalyses(AnalysisSet set)
FunctionAnalyses::FunctionAnalyses(const AnalysisContext& context, const Function& function)
: m_context(context)
, m_function(function)
, m_effectContext { context.program, context.effects ? *context.effects : m_unknownSummaries.emplace() }
, m_effectContext { context.program, context.effects ? *context.effects : m_unknownSummaries.emplace(), context.savedFrames == SavedFrames::AreOutOfReach ? UntestedStorage::LeavesReadKnown : UntestedStorage::MakesReadUnknown }
, m_effects(m_effectContext, function, context.build.options().verifiesAfterEveryPass ? &context.build.diagnostics() : nullptr)
, m_countsBuilds(context.build.options().logging.logsPhaseTimes)
, m_verifiesUpdates(context.build.options().verifiesAfterEveryPass)
Expand Down
73 changes: 73 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTGuardAvailability.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,46 @@ class KillQuery {
return m_exactAccess->kind == MemoryAccess::Kind::NotExact;
}

std::optional<BitVector> allocationsWhoseProofsEnd(WrittenLocations& written)
{
const EscapeAnalysis& escape = m_analyses.escape();
if (written.extent != WriteExtent::AboutSomeObjects || written.isLeftByPropertiesBesideFields || escape.tracksOnlyEntryParameters() || !mayWriteBeyondOperands(written))
return std::nullopt;
if (!m_allocationsNotConfined) {
if (!m_history)
m_history.emplace(m_analyses.escapeHistory(), m_point);
if (m_history->isUnknown())
return std::nullopt;
m_allocationsNotConfined.emplace();
m_history->addObjectsThatMayHaveOccurredTo(*m_allocationsNotConfined);
m_allocationsNotConfined->merge(objectsInsideOperands());
for (size_t object : operandObjects()) {
if (escapesAtWriter(ObjectID(object)))
m_allocationsNotConfined->set(object);
}
}
BitVector allocations = *m_allocationsNotConfined;
for (ValueID operand : operandsWrittenThrough(written))
escape.pointsTo(operand).objects.addTo(allocations);
return allocations;
}

std::optional<Vector<ObjectID, 4>> allocationsWithinReach()
{
if (!m_beyondOperands)
m_beyondOperands = m_writer ? effectsBeyondOperands(m_analyses.effectContext(), m_function, m_function.at(m_writer), m_effects) : m_effects;
if (!m_beyondOperands->onAnyEdge().writes.restrictedTo(perObjectHeaps).isEmpty())
return std::nullopt;
Vector<ObjectID, 4> objects;
for (const WrittenOperand& operand : writtenOperands()) {
std::optional<Vector<ObjectID, 4>> objectsOfOperand = GuardWriter::allocationsOf(m_analyses, operand.operand);
if (!objectsOfOperand)
return std::nullopt;
objects.appendVector(*objectsOfOperand);
}
return objects;
}

bool reachesObjects(WrittenLocations& written)
{
return mayWriteBeyondOperands(written) || !operandsWrittenThrough(written).empty();
Expand Down Expand Up @@ -410,6 +450,7 @@ class KillQuery {
std::optional<Vector<WrittenOperand, 4>> m_writtenOperands;
std::optional<BitVector> m_operandObjects;
std::optional<BitVector> m_objectsInsideOperands;
std::optional<BitVector> m_allocationsNotConfined;
std::optional<UseEffectHistory::AtPoint> m_history;
ValueID m_lastSubject;
bool m_isLastSubjectConfined { false };
Expand Down Expand Up @@ -475,6 +516,38 @@ auto GuardWriter::writtenOf(const DependedLocations& depended) -> WrittenDepende
return { EndedProofs::AboutSomeObjects, static_cast<uint32_t>(m_query->written.size() - 1) };
}

std::optional<BitVector> GuardWriter::allocationsWhoseProofsEnd(const WrittenDependencies& dependencies)
{
if (!m_query || dependencies.endedProofs != EndedProofs::AboutSomeObjects)
return std::nullopt;
return m_query->query.allocationsWhoseProofsEnd(m_query->written[dependencies.index]);
}

std::optional<Vector<ObjectID, 4>> GuardWriter::allocationsWithinReach()
{
if (!m_query)
return std::nullopt;
return m_query->query.allocationsWithinReach();
}

// Two values with one root are one object whatever their sets say, so the root's members are among the answer, and there is at least one.
std::optional<Vector<ObjectID, 4>> GuardWriter::allocationsOf(FunctionAnalyses& analyses, ValueID value)
{
const EscapeAnalysis& escape = analyses.escape();
std::optional<ObjectSet> own = escape.allocationsIfNothingElse(value);
std::optional<ObjectSet> ofRoot = escape.allocationsIfNothingElse(analyses.sameObjectThroughCasts(value).valueInFrontOfFacts());
if (!own || !ofRoot || ofRoot->isEmpty())
return std::nullopt;
Vector<ObjectID, 4> objects;
for (ObjectID object : *ofRoot)
objects.append(object);
for (ObjectID object : *own) {
if (!ofRoot->contains(object))
objects.append(object);
}
return objects;
}

bool GuardWriter::mayEndProofAbout(ValueID subject, bool subjectComesFromOutside, const WrittenDependencies& dependencies)
{
switch (dependencies.endedProofs) {
Expand Down
7 changes: 7 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTGuardAvailability.h
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,13 @@ class GuardWriter {
};
WrittenDependencies writtenOf(const DependedLocations&);
bool mayEndProofAbout(ValueID subject, bool subjectComesFromOutside, const WrittenDependencies&);
// All the objects it may write a location of one object into, where those are allocations of the function. Nothing where it may reach anything else.
std::optional<Vector<ObjectID, 4>> allocationsWithinReach();
// The same for a subject: the allocations it may be. mayEndProofAbout() says no about a subject from inside the function that shares none with the writer.
static std::optional<Vector<ObjectID, 4>> allocationsOf(FunctionAnalyses&, ValueID);
// Of a writer that may write beyond its operands. About a subject from inside the function that can be nothing but allocations of the function, and is at least one, mayEndProofAbout() says yes
// if and only if one of them is here. Nothing where it takes more to tell.
std::optional<BitVector> allocationsWhoseProofsEnd(const WrittenDependencies&);

private:
struct Query;
Expand Down
48 changes: 47 additions & 1 deletion Source/JavaScriptCore/aot/analysis/AOTGuardFacts.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,35 @@ static bool isAnsweredByTestOfCell(const Program& program, const Value& slotOper
});
}

// A field that holds its own object cannot be listed, and the fields behind it wait for the order of the keys. Stores that stand directly behind the allocation, with nothing between that looks at anything, are as good as listed:
// nobody has the object before they have run. To the store that fills the slot it is not filled yet.
static ValueID valueStoredDirectlyBehind(FunctionAnalyses& analyses, ValueID allocation, PropertyKeyID key, const Value& user)
{
const Function& function = analyses.function();
ValueLocation location = analyses.useLists().locationOf(allocation);
const BlockValues& values = function.at(location.block).values;
for (uint32_t index = location.index + 1; index < values.size(); ++index) {
const Value& value = function.at(values[index]);
if (&value == &user)
return { };
if (value.opcode == Opcode::PutSlots && value.objectOperand() == allocation) {
PayloadListView<PlacedKey> keys = value.placedKeys();
std::span<const ValueID> stored = value.variadicOperands();
for (uint32_t position = 0; position < keys.size() && position < stored.size(); ++position) {
if (keys[position].key == key)
return stored[position];
}
continue;
}
bool storesIntoObject = (value.opcode == Opcode::PutSlot || value.opcode == Opcode::PutSlotOrAdd) && value.objectOperand() == allocation;
if (storesIntoObject && value.key() == key)
return value.valueOperand();
if (!storesIntoObject && !analyses.hasNoEffects(values[index]))
return { };
}
return { };
}

static bool fillsSlotWithDeclaredType(FunctionAnalyses& analyses, ValueID allocationID, const Value& user)
{
PropertyKeyID key = user.key();
Expand All @@ -856,6 +885,10 @@ static bool fillsSlotWithDeclaredType(FunctionAnalyses& analyses, ValueID alloca
if (fieldAt(program, { allocation.layout(), fields[index] }).key == key)
listsValue = !mayBeMarker(values[index]);
}
if (!listsValue) {
ValueID stored = valueStoredDirectlyBehind(analyses, allocationID, key, user);
listsValue = stored && !mayBeMarker(stored);
}
if (!listsValue)
return false;

Expand Down Expand Up @@ -1330,6 +1363,19 @@ static constexpr OpcodeSet opcodesBehindProofsOfFilledSlots = opcodesMatching([]
return isTest || (resultRuleOf(opcode) == ResultRule::Allocation && Value::opcodesWithLayout.contains(opcode));
});

static bool isFilledBehindEveryAllocation(FunctionAnalyses& analyses, const GuardFact& fact, const Value& user)
{
PropertyKeyID key = user.key();
// Placement asks of a store whether it finds the slot as a creating store left it, and one of these stores may be the one that asks.
if (storedOf(user.opcode) || (!fact.proves(Proof::SlotFilled) && !fact.proves(Proof::KeyOrder)))
return false;
return std::ranges::all_of(fact.origins, [&](const GuardOrigin& origin) {
if (origin.source != GuardSource::Allocation)
return false;
return listsFieldWithKey(analyses.program(), analyses.function().at(origin.test), key) || valueStoredDirectlyBehind(analyses, origin.test, key, user);
});
}

bool guardProvesSlotFilled(FunctionAnalyses& analyses, const Value& slotOperation)
{
ValueID guard = slotOperation.guardOperand();
Expand All @@ -1346,7 +1392,7 @@ bool guardProvesSlotFilled(FunctionAnalyses& analyses, const Value& slotOperatio
bool isProven = std::ranges::any_of(analyses.guardFacts(guard).facts, [&](const GuardFact& fact) {
if (fact.subject != object)
return false;
return provesSlotFilled(program, analyses.function(), fact, slotOperation.key()) || (testOfCellAnswers && fact.proves(Proof::BirthFieldsFilled));
return provesSlotFilled(program, analyses.function(), fact, slotOperation.key()) || (testOfCellAnswers && fact.proves(Proof::BirthFieldsFilled)) || isFilledBehindEveryAllocation(analyses, fact, slotOperation);
});
RELEASE_ASSERT(mayBeProven || !isProven);
return isProven;
Expand Down
16 changes: 16 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTLiveness.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,22 @@ Liveness::LiveValues Liveness::liveAtHead(BlockID block) const
return LiveValues(m_liveValues.subspan(first, m_firstLiveValue[block.index() + 1] - first));
}

void Liveness::forEachLiveAtHeadOfAny(std::span<const BlockID> blocks, NOESCAPE const ScopedLambda<void(ValueID)>& functor) const
{
if (m_sharedSetAtHead.isEmpty()) [[likely]] {
for (BlockID block : blocks)
forEachLiveAtHead(block, functor);
return;
}
Vector<uint8_t> isVisited(FillWith { }, m_sharedSets.numberOfSets(), static_cast<uint8_t>(0));
for (BlockID block : blocks) {
RELEASE_ASSERT(block.index() < m_cfg.numNodes());
m_sharedSets.forEachInPartsNotVisited(m_sharedSetAtHead[block.index()], isVisited, [&](uint32_t index) {
functor(m_valuesInSharedSets[index]);
});
}
}

bool Liveness::isAnyLiveAtHead(BlockID block) const
{
RELEASE_ASSERT(block.index() < m_cfg.numNodes());
Expand Down
19 changes: 19 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTLiveness.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ class Liveness {
};

LiveValues liveAtHead(BlockID) const LIFETIME_BOUND;
// Each value that is live at the head of one of the blocks, once or more often.
void forEachLiveAtHeadOfAny(std::span<const BlockID>, NOESCAPE const ScopedLambda<void(ValueID)>&) const;
bool isAnyLiveAtHead(BlockID) const;
bool isAnyLiveAtHead(BlockID, const BitVector& values, AnswersAboutValues&) const;
bool isLiveAtHead(ValueID, BlockID) const;
Expand Down Expand Up @@ -176,6 +178,8 @@ class Liveness {
bool contains(Set, uint32_t key) const;
void forEach(Set set, NOESCAPE const Invocable<void(uint32_t)> auto& functor) const { forEach(set, 0, m_numberOfInnerLevels, functor); }
size_t numberOfSets() const { return m_nodes.size(); }
// `isVisited` has an entry for each set, zero at first.
void forEachInPartsNotVisited(Set set, Vector<uint8_t>& isVisited, NOESCAPE const Invocable<void(uint32_t)> auto& functor) const { forEachInPartsNotVisited(set, 0, m_numberOfInnerLevels, isVisited, functor); }
// `answers` has an entry for each set, zero at first, and goes with one functor.
bool containsAny(Set set, Vector<uint8_t>& answers, NOESCAPE const Invocable<bool(uint32_t)> auto& functor) const { return containsAny(set, 0, m_numberOfInnerLevels, answers, functor); }

Expand Down Expand Up @@ -210,6 +214,21 @@ class Liveness {
}
}

void forEachInPartsNotVisited(Set set, uint32_t firstKey, unsigned level, Vector<uint8_t>& isVisited, NOESCAPE const Invocable<void(uint32_t)> auto& functor) const
{
if (!set || std::exchange(isVisited[set], 1))
return;
const Node& node = m_nodes[set];
for (unsigned slot = 0; slot < slotsPerNode; ++slot) {
if (level) {
forEachInPartsNotVisited(node[slot], firstKey + (slot << (bitsPerLeafKey + bitsPerSlotIndex * (level - 1))), level - 1, isVisited, functor);
continue;
}
for (uint32_t bits = node[slot]; bits; bits &= bits - 1)
functor(firstKey + slot * bitsPerLeafWord + std::countr_zero(bits));
}
}

bool containsAny(Set set, uint32_t firstKey, unsigned level, Vector<uint8_t>& answers, NOESCAPE const Invocable<bool(uint32_t)> auto& functor) const
{
constexpr uint8_t notAsked = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -726,7 +726,7 @@ MemberAnswer findInstanceMember(const PropertyAccessContext& context, const Memb
// The image that made the object holds the slot to a type of its own and keeps the marks of its cells: a store goes by name.
if (storesValue && program.at(query.layout).has(LayoutFlags::IsOfImageBelow))
return MemberAnswer::unknown(UnprovenCause::Unknown);
return { .kind = MemberKind::Field, .level = KnowledgeLevel::Proven, .holder = found->layout, .field = found->field };
return { .kind = MemberKind::Field, .level = KnowledgeLevel::Proven, .holder = found->layout, .field = found->field, .isAccessorInSlot = fieldAt(program, *found).form.contains(FieldForm::IsAccessor) };
}

// What an object of another image has beside its fields, and what stands above it, that image alone can say.
Expand Down
2 changes: 2 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTPropertyAccessMembers.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ struct MemberAnswer {
KnowledgeLevel level { KnowledgeLevel::Unknown };
LayoutID holder;
uint32_t field { 0 };
// The slot of the field holds a getter, a setter or both.
bool isAccessorInSlot { false };
FunctionID method;
FunctionID getter;
FunctionID setter;
Expand Down
15 changes: 11 additions & 4 deletions Source/JavaScriptCore/aot/analysis/AOTPropertyAccessPartition.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ static bool haveSameMember(const MemberAnswer& a, const MemberAnswer& b)
case MemberKind::Unknown:
return false;
case MemberKind::Absent:
case MemberKind::Field:
return true;
case MemberKind::Field:
return a.isAccessorInSlot == b.isAccessorInSlot;
case MemberKind::Method:
return a.method == b.method;
case MemberKind::Accessor:
Expand Down Expand Up @@ -104,7 +105,9 @@ static uint64_t hashOfArm(const MemberAnswer& answer)
switch (answer.kind) {
case MemberKind::Unknown:
case MemberKind::Absent:
break;
case MemberKind::Field:
member = answer.isAccessorInSlot;
break;
case MemberKind::Method:
member = answer.method.index();
Expand Down Expand Up @@ -220,6 +223,7 @@ static auto withAnswersFor(const PropertyAccessContext& context, std::span<const
MemberAnswer answer = fieldAnswer->second;
answer.holder = field->layout;
answer.field = field->field;
answer.isAccessorInSlot = fieldAt(program, *field).form.contains(FieldForm::IsAccessor);
return answer;
}
if (field) {
Expand Down Expand Up @@ -282,7 +286,7 @@ static void forEachNumberOf(auto& answer, NOESCAPE const auto& functor)
static uint32_t wordOf(uint32_t number) { return number; }
static uint32_t wordOf(const auto& id) { return id.index(); }

// How many arms; how many layouts are the rest; those layouts. Then each arm: kind, level, cause and which numbers follow; how many layouts, and whether proofs follow; how many conditions;
// How many arms; how many layouts are the rest; those layouts. Then each arm: kind, level, cause and which numbers follow; how many layouts, and whether proofs follow; how many conditions, and whether the slot holds an accessor;
// the proofs; the numbers; the layouts; the conditions.
PackedMemberPartition::PackedMemberPartition(const MemberPartition& partition)
{
Expand All @@ -304,7 +308,8 @@ PackedMemberPartition::PackedMemberPartition(const MemberPartition& partition)
size_t first = words.size();
words.append(static_cast<uint32_t>(answer.kind) | static_cast<uint32_t>(answer.level) << 8 | static_cast<uint32_t>(answer.cause) << 16);
words.append(static_cast<uint32_t>(arm.layouts.size()) << 1 | static_cast<uint32_t>(hasProofs));
words.append(static_cast<uint32_t>(answer.conditions.size()));
RELEASE_ASSERT(answer.conditions.size() <= UINT32_MAX >> 1);
words.append(static_cast<uint32_t>(answer.conditions.size()) << 1 | static_cast<uint32_t>(answer.isAccessorInSlot));
if (hasProofs) {
uint64_t proofs = std::bit_cast<uint64_t>(answer.receiverProofs);
words.append(static_cast<uint32_t>(proofs));
Expand Down Expand Up @@ -346,7 +351,9 @@ MemberPartition PackedMemberPartition::unpack() const
MemberAnswer& answer = arm.answer;
uint32_t first = words[next++];
uint32_t second = words[next++];
uint32_t numberOfConditions = words[next++];
uint32_t third = words[next++];
uint32_t numberOfConditions = third >> 1;
answer.isAccessorInSlot = third & 1;
answer.kind = static_cast<MemberKind>(first & 0xff);
answer.level = static_cast<KnowledgeLevel>(first >> 8 & 0xff);
answer.cause = static_cast<UnprovenCause>(first >> 16 & 0xff);
Expand Down
Loading
Loading