Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
4 changes: 4 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTAnalysisContext.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ class FlowResults;
enum class CallResults : uint8_t { MayAliasOperands, NotFollowed };
// The verifier judges code that the passes have finished, so a limit that the build lowers stays as usual for it.
enum class WorkLimits : bool { OfBuild, NeverLowered };
// A saved frame is out of the program's reach, so what a suspension holds has gone nowhere. Only the verifier judges by that: a pass that kept a guard by it would keep it for a stage that
// reads the object out of a frame in the heap, where it comes from outside.
enum class SavedFrames : bool { AreUnknownMemory, AreOutOfReach };

struct AnalysisContext {
BuildContext& build;
Expand All @@ -31,6 +34,7 @@ struct AnalysisContext {
FunctionID id { };
CallResults callResults { CallResults::MayAliasOperands };
WorkLimits workLimits { WorkLimits::OfBuild };
SavedFrames savedFrames { SavedFrames::AreUnknownMemory };

uint64_t limit(LimitID) const;
};
Expand Down
57 changes: 35 additions & 22 deletions Source/JavaScriptCore/aot/analysis/AOTCallGraph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -304,16 +304,13 @@ void BodyReferences::didChange(std::span<const FunctionID> functions)
}
}

static BitVector findFunctionsExpectedByCode(const Program& program, const Vector<ReferencesOfBody>& bodies)
static BitVector findFunctionsExpectedByCode(const Program& program, const Vector<uint8_t>& isExpectedType)
{
BitVector isListed(program.types.size());
for (const ReferencesOfBody& body : bodies) {
for (TypeID type : body.expectedFunctionTypes)
isListed.set(type.index());
}
BitVector functions(program.functions.size());
for (size_t type : isListed) {
for (FunctionID function : program.at(TypeID(static_cast<uint32_t>(type))).functions)
for (uint32_t type = 0; type < isExpectedType.size(); ++type) {
if (!isExpectedType[type])
continue;
for (FunctionID function : program.at(TypeID(type)).functions)
functions.set(function.index());
}
return functions;
Expand Down Expand Up @@ -988,6 +985,7 @@ CallGraph buildCallGraph(const Program& program, const FlowResults& flow, BuildC
AOT_THRESHOLD(uint32_t, minCalleesOfLongList, 16);
Vector<unsigned> hashOfLongList(FillWith { }, count, 0u);
Vector<uint8_t> hasLongList(FillWith { }, count, static_cast<uint8_t>(0));
Vector<uint8_t> isExpectedType(FillWith { }, program.types.size(), static_cast<uint8_t>(0));
std::atomic<bool> someFunctionHasLongList { false };
constexpr unsigned prefetchDistance = 4;
forEachIndexInParallel(loops, count, [&](unsigned index) {
Expand Down Expand Up @@ -1020,6 +1018,10 @@ CallGraph buildCallGraph(const Program& program, const FlowResults& flow, BuildC
WTF::atomicExchangeAdd(&calleeSetCallSites[set.index()], 1u, std::memory_order_relaxed);
for (CalleeSetID set : body.enteredSets)
WTF::atomicExchangeAdd(&enteredSetCallSites[set.index()], 1u, std::memory_order_relaxed);
for (TypeID type : body.expectedFunctionTypes) {
if (!WTF::atomicLoad(&isExpectedType[type.index()], std::memory_order_relaxed))
WTF::atomicStore(&isExpectedType[type.index()], static_cast<uint8_t>(1), std::memory_order_relaxed);
}
});
std::partial_sum(calleeOffsets.begin(), calleeOffsets.end(), calleeOffsets.begin());
callees.grow(calleeOffsets[count]);
Expand Down Expand Up @@ -1083,21 +1085,32 @@ CallGraph buildCallGraph(const Program& program, const FlowResults& flow, BuildC
add(callee.index(), set);
}
});
groupByIndex(count, lists.firstSetOfUser, lists.setsOfUsers, [&](const auto& add) {
for (unsigned index = 0; index < count; ++index) {
Vector<uint32_t, 8> setsOfUser;
for (const CompactList<CalleeSetID>* sets : { &bodies.m_bodies[index].calleeSets, &bodies.m_bodies[index].enteredSets }) {
CalleeSetID previous;
for (CalleeSetID set : *sets) {
if (set != std::exchange(previous, set) && program.at(set).isLong())
setsOfUser.append(static_cast<uint32_t>(std::ranges::lower_bound(longSets, set.index()) - longSets.begin()));
}
constexpr uint32_t isNoLongSet = UINT32_MAX;
Vector<uint32_t> positionOfLongSet(FillWith { }, calleeSetCallSites.size(), isNoLongSet);
for (uint32_t position = 0; position < longSets.size(); ++position)
positionOfLongSet[longSets[position]] = position;
auto longSetsOfUser = [&](unsigned index) {
Vector<uint32_t, 8> setsOfUser;
for (const CompactList<CalleeSetID>* sets : { &bodies.m_bodies[index].calleeSets, &bodies.m_bodies[index].enteredSets }) {
CalleeSetID previous;
for (CalleeSetID set : *sets) {
if (set != std::exchange(previous, set) && positionOfLongSet[set.index()] != isNoLongSet)
setsOfUser.append(positionOfLongSet[set.index()]);
}
AOT::sort(setsOfUser);
removeRepeatedElements(setsOfUser);
for (uint32_t set : setsOfUser)
add(index, set);
}
AOT::sort(setsOfUser);
removeRepeatedElements(setsOfUser);
return setsOfUser;
};
lists.firstSetOfUser.fill(0, count + 1);
forEachIndexInParallel(loops, count, [&](unsigned index) {
lists.firstSetOfUser[index + 1] = longSetsOfUser(index).size();
});
std::partial_sum(lists.firstSetOfUser.begin(), lists.firstSetOfUser.end(), lists.firstSetOfUser.begin());
lists.setsOfUsers.grow(lists.firstSetOfUser[count]);
forEachIndexInParallel(loops, count, [&](unsigned index) {
if (lists.firstSetOfUser[index] != lists.firstSetOfUser[index + 1])
std::ranges::copy(longSetsOfUser(index), lists.setsOfUsers.begin() + lists.firstSetOfUser[index]);
});
groupByIndex(longSets.size(), lists.firstUser, lists.users, [&](const auto& add) {
for (unsigned index = 0; index < count; ++index) {
Expand Down Expand Up @@ -1186,7 +1199,7 @@ CallGraph buildCallGraph(const Program& program, const FlowResults& flow, BuildC
graph.m_hasCompleteCallSiteSet = createBitVector(hasCompleteCallSiteSet);
},
[&] {
graph.m_isExpectedByCode = findFunctionsExpectedByCode(program, bodies.m_bodies);
graph.m_isExpectedByCode = findFunctionsExpectedByCode(program, isExpectedType);
},
});
times.didFinish("listCallersAndRoots"_s);
Expand Down
8 changes: 7 additions & 1 deletion Source/JavaScriptCore/aot/analysis/AOTEscapeAnalysis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ uint64_t EscapeAnalysis::solve()
computeUses(results, inputs.useLists, useReasons.span());
computeStates();
takeSets(results);
m_objectsInLocalHeapWords = WTF::move(results.objectsInLocalHeapWords);
if (m_tracksOnlyEntryParameters)
m_liveInstances.fill(LiveInstances::Unknown, m_objects.size());
m_remembersAnswersFromTypes = inputs.remembersAnswersFromTypes;
Expand Down Expand Up @@ -617,6 +618,8 @@ bool mayNameUnescapedAllocation(const Function& function, ValueID value)
}
if (definition.opcode != Opcode::GetHeapFrameWord && definingOpcodes.contains(definition.opcode))
return true;
if (accessesLocalHeapWords(function, definition))
return true;
definition.forEachOperand([&](ValueID operand, uint32_t) {
definitions.appendIfNotContains(operand);
});
Expand Down Expand Up @@ -655,6 +658,8 @@ static BitVector collectWatchedValues(const EscapeAnalysis& escape, std::span<co
static LiveInstances computeLiveInstancesAt(const EscapeAnalysis& escape, const Liveness::LocalCalc& liveBeforeAllocation, const WatchedObjects& watchedObjects)
{
for (ObjectID object : watchedObjects.objects()) {
if (escape.isInLocalHeapWords(object))
return LiveInstances::Unknown;
for (ValueID alias : escape.aliasesOf(object)) {
if (liveBeforeAllocation.isLive(alias))
return LiveInstances::Unknown;
Expand Down Expand Up @@ -775,7 +780,8 @@ EscapeAnalysis::AllocationSet EscapeAnalysis::allocationSetOf(ValueID value) con
ValueSetID id = m_setOfValue[value.index()];
if (!id) {
// The function that resumes a generator reads the words of the frame before it stores any, and returns behind its stores: what a word holds an earlier run has put there.
bool isWordOfOwnFrame = m_function.role == FunctionRole::CoroutineBody && m_function.at(sameObjectThroughCasts(m_function, value).valueInFrontOfFacts()).opcode == Opcode::GetHeapFrameWord;
const Value& definition = m_function.at(sameObjectThroughCasts(m_function, value).valueInFrontOfFacts());
bool isWordOfOwnFrame = m_function.role == FunctionRole::CoroutineBody && definition.opcode == Opcode::GetHeapFrameWord && !accessesLocalHeapWords(m_function, definition);
return { m_emptySet, true, isWordOfOwnFrame, nullptr };
}
const StoredSet& set = m_sets[id.index()];
Expand Down
4 changes: 4 additions & 0 deletions Source/JavaScriptCore/aot/analysis/AOTEscapeAnalysis.h
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,7 @@ class EscapeAnalysis {
}
std::span<const ObjectID> contentsOf(ObjectID object) const LIFETIME_BOUND { return entriesOf(object, m_contents, m_firstContent); }
std::span<const ObjectID> containersOf(ObjectID object) const LIFETIME_BOUND { return entriesOf(object, m_containers, m_firstContainer); }
bool isInLocalHeapWords(ObjectID object) const { return m_objectsInLocalHeapWords.contains(object); }

AliasResult alias(ValueID, ValueID) const;
bool mayAlias(ValueID, ValueID) const;
Expand Down Expand Up @@ -395,6 +396,7 @@ class EscapeAnalysis {
Vector<StoredSet> m_sets;
BitVector m_noObjects;
SparseObjectSet m_emptySet;
SparseObjectSet m_objectsInLocalHeapWords;
mutable Vector<ValueID> m_aliases;
mutable Vector<uint32_t> m_firstAlias;
// Two entries per use that the solver recorded: not an escape, then an escape. That is all that differs between the objects of one use.
Expand Down Expand Up @@ -423,6 +425,8 @@ class EscapeAnalysis {

// Words of different groups hold different objects.
bool isFrameWordWithAllocationGroup(const Function&, const Value&);
// A load or store of a word in a NewHeapWords cell of this function.
bool accessesLocalHeapWords(const Function&, const Value&);

// False where every allocation that the value may name was read out of an object that is open to others, and so has a use that lets it escape.
bool mayNameUnescapedAllocation(const Function&, ValueID);
Expand Down
66 changes: 39 additions & 27 deletions Source/JavaScriptCore/aot/analysis/AOTFamilyTables.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,7 @@ AOT_THRESHOLD(size_t, minCachedSetSize, 8);

static LayoutList listFromSorted(std::span<const LayoutID> sortedLayouts)
{
LayoutList list;
for (LayoutID layout : sortedLayouts)
list.add(layout);
return list;
return LayoutList::fromAscending(sortedLayouts);
}

static std::span<const LayoutID> membersListedIn(const Program& program, LayoutSetID set)
Expand All @@ -139,11 +136,16 @@ static std::span<const LayoutID> membersListedIn(const Program& program, LayoutS
return program.at(set).written().span();
}

static bool isAllocated(const Program& program, LayoutID layout)
{
return program.at(layout).flow.allocation != AllocationState::NeverAllocated;
}

static LayoutList computeAllocatedMembers(const Program& program, std::span<const LayoutID> members)
{
LayoutList allocatedMembers;
for (LayoutID layout : members) {
if (program.at(layout).flow.allocation != AllocationState::NeverAllocated)
if (isAllocated(program, layout))
allocatedMembers.add(layout);
}
return allocatedMembers;
Expand All @@ -160,12 +162,15 @@ struct FamilyTables::Cache {
WTF_DEPRECATED_MAKE_STRUCT_FAST_ALLOCATED(Cache);

struct Family {
std::span<const LayoutID> members() const LIFETIME_BOUND { return hasOnlyAllocatedMembers ? allocatedMembers.span() : allMembers.span(); }
std::span<const LayoutID> members() const LIFETIME_BOUND { return membersInTable.data() ? membersInTable : copyOfMembers.span(); }
std::span<const LayoutID> allocatedMembers() const LIFETIME_BOUND { return hasOnlyAllocatedMembers ? members() : allocatedAmongMembers.span(); }

uint32_t number { 0 };
bool hasOnlyAllocatedMembers { true };
Vector<LayoutID> allMembers;
LayoutList allocatedMembers;
// A set of the table stays where it is for as long as the program.
std::span<const LayoutID> membersInTable;
Vector<LayoutID> copyOfMembers;
LayoutList allocatedAmongMembers;
ReceiverState state;
};

Expand Down Expand Up @@ -194,10 +199,15 @@ struct FamilyTables::Cache {

Family family;
family.state = receiverStateOf(program, members);
family.allocatedMembers = computeAllocatedMembers(program, members);
family.hasOnlyAllocatedMembers = family.allocatedMembers.size() == members.size();
family.hasOnlyAllocatedMembers = std::ranges::all_of(members, [&](LayoutID layout) {
return isAllocated(program, layout);
});
if (!family.hasOnlyAllocatedMembers)
family.allMembers = Vector<LayoutID>(members);
family.allocatedAmongMembers = computeAllocatedMembers(program, members);
if (TypeTable::isProvisional(set))
family.copyOfMembers = Vector<LayoutID>(members);
else
family.membersInTable = members;
family.number = numberOfFamilies.fetch_add(1, std::memory_order_relaxed);
return &families.add(hash, WTF::move(family), hasSameMembers);
});
Expand Down Expand Up @@ -248,7 +258,7 @@ auto FamilyTables::replaceCache(uint64_t phase) const -> Cache&

std::span<const LayoutID> FamilyTables::allocatedMembersOf(LayoutSetID set) const
{
return cache().familyOf(m_program, set).allocatedMembers.span();
return cache().familyOf(m_program, set).allocatedMembers();
}

ReceiverState FamilyTables::stateOf(LayoutSetID set) const
Expand All @@ -261,7 +271,7 @@ ReceiverState FamilyTables::stateOf(LayoutSetID set) const
return cache().familyOf(m_program, set).state;
}

static FamilyMember computeMember(const Program& program, const LayoutList& allocatedMembers, PropertyKeyID key)
static FamilyMember computeMember(const Program& program, std::span<const LayoutID> allocatedMembers, PropertyKeyID key)
{
FamilyMember member;
Vector<LayoutID> holders;
Expand Down Expand Up @@ -295,52 +305,54 @@ const FamilyMember& FamilyTables::memberOf(LayoutSetID set, PropertyKeyID key) c
Cache& cache = this->cache();
const Cache::Family& family = cache.familyOf(m_program, set);
return cache.members.ensure(computeHash(family.number, key), { family.number, key }, [&] {
return computeMember(m_program, family.allocatedMembers, key);
return computeMember(m_program, family.allocatedMembers(), key);
});
}

auto FamilyTables::candidatesOf(LayoutSetID receiver, LayoutSetID expected, LayoutSetID stamp) const -> Candidates
{
if (expected == receiver)
expected = { };
auto predictedMembers = [&](const LayoutList& proven, const LayoutList& expectedMembers) -> Expected<LayoutList> {
auto predictedMembers = [&](std::span<const LayoutID> proven, std::span<const LayoutID> expectedMembers) -> Expected<LayoutList> {
const LayoutSet* stamped = stamp ? &m_program.at(stamp) : nullptr;
LayoutList predicted;
if (!receiver) {
if (expected)
predicted = expectedMembers;
predicted = listFromSorted(expectedMembers);
if (stamped)
predicted.unite(listFromSorted(stamped->seen().predicted()));
return predicted;
}
if (!expected && !(stamped && stamped->listsEveryLayout()))
return predicted;

predicted = proven;
predicted = listFromSorted(proven);
if (expected)
predicted.intersect(expectedMembers);
predicted.intersect(listFromSorted(expectedMembers));
if (stamped && stamped->listsEveryLayout())
predicted.intersect(listFromSorted(stamped->completeMembers()));
if (predicted.size() == proven.size())
predicted.clear();
return predicted;
};
auto membersOfExpectedType = [&](const LayoutList& expectedMembers) -> Expected<LayoutList> {
return receiver || !expected ? LayoutList() : expectedMembers;
auto membersOfExpectedType = [&](std::span<const LayoutID> expectedMembers) -> Expected<LayoutList> {
return receiver || !expected ? LayoutList() : listFromSorted(expectedMembers);
};

if (std::max({ membersListedIn(m_program, receiver).size(), membersListedIn(m_program, expected).size(), membersListedIn(m_program, stamp).size() }) < minCachedSetSize) {
LayoutList proven = computeAllocatedMembers(m_program, membersListedIn(m_program, receiver));
LayoutList expectedMembers = computeAllocatedMembers(m_program, membersListedIn(m_program, expected));
Expected<LayoutList> predicted = predictedMembers(proven, expectedMembers);
return { WTF::move(proven), WTF::move(predicted), membersOfExpectedType(expectedMembers) };
Expected<LayoutList> predicted = predictedMembers(proven.span(), expectedMembers.span());
return { WTF::move(proven), WTF::move(predicted), membersOfExpectedType(expectedMembers.span()) };
}

Cache& cache = this->cache();
const Cache::Candidates& known = cache.candidates.ensure(computeHash(receiver, expected, stamp), { receiver, expected, stamp }, [&] {
const Cache::Family& receiverFamily = cache.familyOf(m_program, receiver);
const LayoutList& expectedMembers = cache.familyOf(m_program, expected).allocatedMembers;
return Cache::Candidates { &receiverFamily, predictedMembers(receiverFamily.allocatedMembers, expectedMembers), membersOfExpectedType(expectedMembers) };
std::span<const LayoutID> expectedMembers = cache.familyOf(m_program, expected).allocatedMembers();
return Cache::Candidates { &receiverFamily, predictedMembers(receiverFamily.allocatedMembers(), expectedMembers), membersOfExpectedType(expectedMembers) };
});
return { known.receiverFamily->allocatedMembers, known.expected, known.ofExpectedType };
return { listFromSorted(known.receiverFamily->allocatedMembers()), known.expected, known.ofExpectedType };
}

bool FamilyTables::isSubsetOf(LayoutSetID subset, LayoutSetID superset) const
Expand All @@ -360,7 +372,7 @@ bool FamilyTables::isSubsetOf(LayoutSetID subset, LayoutSetID superset) const
return cache().subsets.ensure(computeHash(subset, superset), { subset, superset }, compute);
}

static Vector<FieldOfMember> computeFieldsOfMembers(const Program& program, const LayoutList& allocatedMembers, PropertyKeyID key, TypeLevel level)
static Vector<FieldOfMember> computeFieldsOfMembers(const Program& program, std::span<const LayoutID> allocatedMembers, PropertyKeyID key, TypeLevel level)
{
Vector<FieldOfMember> fields;
for (LayoutID layout : allocatedMembers) {
Expand Down Expand Up @@ -407,7 +419,7 @@ LayoutSetID FamilyTables::membersWhoseFieldCannotHold(LayoutSetID set, PropertyK
const Cache::Family& family = cache.familyOf(m_program, set);
return cache.excludedMembers.ensure(computeHash(family.number, key, read, level), { family.number, key, read, level }, [&] {
const Vector<FieldOfMember>& fields = cache.fieldsOfMembers.ensure(computeHash(family.number, key, level), { family.number, key, level }, [&] {
return computeFieldsOfMembers(m_program, family.allocatedMembers, key, level);
return computeFieldsOfMembers(m_program, family.allocatedMembers(), key, level);
});
return computeMembersWhoseFieldCannotHold(m_program, fields.span(), read);
});
Expand Down
3 changes: 0 additions & 3 deletions Source/JavaScriptCore/aot/analysis/AOTFunctionAnalyses.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,6 @@ const OpcodeEffectFacts& effectFactsOf(Opcode opcode)

const OpcodeEffectFacts& effectFactsOf(const Value& value)
{
static const OpcodeEffectFacts factsOfObjectWithPrototypeOfLayout = computeEffectFactsOf(Opcode::NewObject, OptionalOperands::AreOmitted);
if (value.opcode == Opcode::NewObject && !value.prototypeOperand())
return factsOfObjectWithPrototypeOfLayout;
return effectFactsOf(value.opcode);
}

Expand Down
Loading
Loading