diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java index d61d3d0690e1..78115d9eb3f3 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/PEGraphDecoder.java @@ -344,7 +344,7 @@ public String getNodeClassName() { } protected class PENonAppendGraphBuilderContext extends CoreProvidersDelegate implements GraphBuilderContext { - protected final PEMethodScope methodScope; + public final PEMethodScope methodScope; protected final Invoke invoke; @Override diff --git a/substratevm/CHANGELOG.md b/substratevm/CHANGELOG.md index 9542ed34d5be..9560988d0e20 100644 --- a/substratevm/CHANGELOG.md +++ b/substratevm/CHANGELOG.md @@ -18,6 +18,7 @@ This changelog summarizes major changes to GraalVM Native Image. * (GR-30433) Disallow the deprecated environment variable USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM=false. * (GR-49655) Experimental support for parts of the [Foreign Function & Memory API](https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/ForeignInterface.md) (part of "Project Panama", [JEP 454](https://openjdk.org/jeps/454)) on AMD64. Must be enabled with `-H:+ForeignAPISupport` (requiring `-H:+UnlockExperimentalVMOptions`). * (GR-46407) Correctly rethrow build-time linkage errors at run-time for registered reflection queries. +* (GR-51002) Improve intrinsification of method handles. This especially improves the performance of `equals` and `hashCode` methods for records, which use method handles that are now intrinsified. ## GraalVM for JDK 21 (Internal Version 23.1.0) * (GR-35746) Lower the default aligned chunk size from 1 MB to 512 KB for the serial and epsilon GCs, reducing memory usage and image size in many cases. diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ObjectScanner.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ObjectScanner.java index 7ed9916b5c32..7e09be8fa42f 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ObjectScanner.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/ObjectScanner.java @@ -145,7 +145,7 @@ protected void scanEmbeddedRoot(JavaConstant root, Object position) { try { scanningObserver.forEmbeddedRoot(root, reason); scanConstant(root, reason); - } catch (UnsupportedFeatureException ex) { + } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) { bb.getUnsupportedFeatures().addMessage(reason.toString(), reason.getMethod(), ex.getMessage(), null, ex); } } @@ -205,7 +205,7 @@ protected void scanField(AnalysisField field, JavaConstant receiver, ScanReason scanningObserver.forPrimitiveFieldValue(receiver, field, fieldValue, reason); } - } catch (UnsupportedFeatureException ex) { + } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) { unsupportedFeatureDuringFieldScan(bb, field, receiver, ex, reason); } } @@ -268,7 +268,7 @@ protected final void scanArray(JavaConstant array, ScanReason prevReason) { try { JavaConstant element = bb.getUniverse().getSnippetReflection().forObject(bb.getUniverse().replaceObject(e)); scanArrayElement(array, arrayType, reason, idx, element); - } catch (UnsupportedFeatureException ex) { /* Object replacement can throw. */ + } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) { unsupportedFeatureDuringConstantScan(bb, bb.getUniverse().getSnippetReflection().forObject(e), ex, reason); } } @@ -313,7 +313,7 @@ public void scanConstant(JavaConstant value, ScanReason reason) { * Use the constant hashCode as a key for the unsupported feature to register only one error * message if the constant is reachable from multiple places. */ - public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant constant, UnsupportedFeatureException e, ScanReason reason) { + public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant constant, Throwable e, ScanReason reason) { unsupportedFeature(bb, String.valueOf(receiverHashCode(constant)), e.getMessage(), reason); } @@ -322,11 +322,11 @@ public static void unsupportedFeatureDuringConstantScan(BigBang bb, JavaConstant * only one error message if the value is reachable from multiple places. For example both the * heap scanning and the heap verification would scan a field that contains an illegal value. */ - public static void unsupportedFeatureDuringFieldScan(BigBang bb, AnalysisField field, JavaConstant receiver, UnsupportedFeatureException e, ScanReason reason) { + public static void unsupportedFeatureDuringFieldScan(BigBang bb, AnalysisField field, JavaConstant receiver, Throwable e, ScanReason reason) { unsupportedFeature(bb, (receiver != null ? receiverHashCode(receiver) + "_" : "") + field.format("%H.%n"), e.getMessage(), reason); } - public static void unsupportedFeatureDuringFieldFolding(BigBang bb, AnalysisField field, JavaConstant receiver, UnsupportedFeatureException e, AnalysisMethod parsedMethod, int bci) { + public static void unsupportedFeatureDuringFieldFolding(BigBang bb, AnalysisField field, JavaConstant receiver, Throwable e, AnalysisMethod parsedMethod, int bci) { ScanReason reason = new FieldConstantFold(field, parsedMethod, bci, receiver, new MethodParsing(parsedMethod)); unsupportedFeature(bb, (receiver != null ? receiverHashCode(receiver) + "_" : "") + field.format("%H.%n"), e.getMessage(), reason); } @@ -448,7 +448,7 @@ private void doScan(WorklistEntry entry) { /* Scan the array elements. */ scanArray(entry.constant, entry.reason); } - } catch (UnsupportedFeatureException ex) { + } catch (UnsupportedFeatureException | AnalysisError.TypeNotFoundError ex) { unsupportedFeatureDuringConstantScan(bb, entry.constant, ex, entry.reason); } } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java index b71030577aa5..939849b0368c 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/heap/ImageHeapScanner.java @@ -595,6 +595,9 @@ public void rescanField(Object receiver, Field reflectionField, ScanReason reaso if (type.isReachable()) { AnalysisField field = metaAccess.lookupJavaField(reflectionField); assert !field.isStatic() : field; + if (!field.isReachable()) { + return; + } JavaConstant receiverConstant = asConstant(receiver); Optional replaced = maybeReplace(receiverConstant, reason); if (replaced.isPresent()) { diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java index 308b33edc49c..9857f6fbcf44 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisGraphDecoder.java @@ -38,6 +38,7 @@ import com.oracle.graal.pointsto.meta.AnalysisField; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.meta.HostedProviders; +import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy.AbstractPolicyScope; import com.oracle.graal.pointsto.util.AnalysisError; import com.oracle.graal.pointsto.util.GraalAccess; import com.oracle.svm.util.ReflectionUtil; @@ -60,6 +61,7 @@ import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.calc.IsNullNode; import jdk.graal.compiler.nodes.extended.UnsafeAccessNode; +import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin; import jdk.graal.compiler.nodes.graphbuilderconf.LoopExplosionPlugin; @@ -78,7 +80,7 @@ public class InlineBeforeAnalysisGraphDecoder extends PEGraphDecoder { public class InlineBeforeAnalysisMethodScope extends PEMethodScope { - public final InlineBeforeAnalysisPolicy.AbstractPolicyScope policyScope; + public final AbstractPolicyScope policyScope; private boolean inliningAborted; @@ -103,7 +105,7 @@ public class InlineBeforeAnalysisMethodScope extends PEMethodScope { for (int i = 0; i < arguments.length; i++) { constArgsWithReceiver[i] = arguments[i].isConstant(); } - policyScope = policy.openCalleeScope(cast(caller).policyScope, bb.getMetaAccess(), method, constArgsWithReceiver, invokeData.intrinsifiedMethodHandle); + policyScope = policy.openCalleeScope(cast(caller).policyScope, method); if (graph.getDebug().isLogEnabled()) { graph.getDebug().logv(" ".repeat(inliningDepth) + "openCalleeScope for " + method.format("%H.%n(%p)") + ": " + policyScope); } @@ -121,6 +123,27 @@ static void recordInlined(InlineBeforeAnalysisMethodScope callerScope, InlineBef } } + static final class InlineBeforeAnalysisInlineInvokePlugin implements InlineInvokePlugin { + + private final InlineBeforeAnalysisPolicy policy; + + InlineBeforeAnalysisInlineInvokePlugin(InlineBeforeAnalysisPolicy policy) { + this.policy = policy; + } + + @Override + public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod m, ValueNode[] args) { + AnalysisMethod method = (AnalysisMethod) m; + + AbstractPolicyScope policyScope = cast(((PENonAppendGraphBuilderContext) b).methodScope).policyScope; + if (policy.shouldInlineInvoke(b, policyScope, method, args)) { + return policy.createInvokeInfo(method); + } else { + return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; + } + } + } + private Field dmhStaticAccessorOffsetField; private Field dmhStaticAccessorBaseField; private AnalysisField dmhStaticAccessorOffsetAnalysisField; @@ -301,8 +324,19 @@ private void ensureDMHStaticAccessorFieldsInitialized() { } @Override - protected void handleNonInlinedInvoke(MethodScope methodScope, LoopScope loopScope, InvokeData invokeData) { + protected void handleNonInlinedInvoke(MethodScope ms, LoopScope loopScope, InvokeData invokeData) { + InlineBeforeAnalysisMethodScope methodScope = cast(ms); maybeAbortInlining(methodScope, loopScope, invokeData.invoke.asNode()); + + if (!methodScope.inliningAborted && methodScope.isInlinedMethod()) { + if (graph.getDebug().isLogEnabled()) { + graph.getDebug().logv(" ".repeat(methodScope.inliningDepth) + " nonInlinedInvoke " + invokeData.callTarget.targetMethod() + ": " + methodScope.policyScope); + } + if (!methodScope.policyScope.processNonInlinedInvoke(providers, invokeData.callTarget)) { + abortInlining(methodScope); + } + } + super.handleNonInlinedInvoke(methodScope, loopScope, invokeData); } @@ -494,7 +528,7 @@ private void killControlFlowNodes(PEMethodScope inlineScope, FixedNode start) { * at the cost of this ugly cast. */ @SuppressWarnings("unchecked") - protected InlineBeforeAnalysisMethodScope cast(MethodScope methodScope) { + protected static InlineBeforeAnalysisMethodScope cast(MethodScope methodScope) { return (InlineBeforeAnalysisMethodScope) methodScope; } diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java deleted file mode 100644 index d24fe0429d9e..000000000000 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisInlineInvokePlugin.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.pointsto.phases; - -import com.oracle.graal.pointsto.meta.AnalysisMethod; - -import jdk.graal.compiler.nodes.ValueNode; -import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; -import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin; -import jdk.vm.ci.meta.ResolvedJavaMethod; - -final class InlineBeforeAnalysisInlineInvokePlugin implements InlineInvokePlugin { - - private final InlineBeforeAnalysisPolicy policy; - - InlineBeforeAnalysisInlineInvokePlugin(InlineBeforeAnalysisPolicy policy) { - this.policy = policy; - } - - @Override - public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod m, ValueNode[] args) { - AnalysisMethod method = (AnalysisMethod) m; - if (policy.shouldInlineInvoke(b, method, args)) { - return policy.createInvokeInfo(method); - } else { - return InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; - } - } -} diff --git a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java index c13fad407ee2..4d17a00af129 100644 --- a/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java +++ b/substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/phases/InlineBeforeAnalysisPolicy.java @@ -30,11 +30,13 @@ import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.graph.NodeSourcePosition; +import jdk.graal.compiler.nodes.CallTargetNode; import jdk.graal.compiler.nodes.FixedWithNextNode; import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin; +import jdk.graal.compiler.nodes.spi.CoreProviders; /** * Provides the policy which methods are inlined by {@link InlineBeforeAnalysis}. If @@ -77,6 +79,8 @@ protected AbstractPolicyScope(int inliningDepth) { * decision on the current list of usages. The list of usages is often but not always empty. */ public abstract boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, Node node); + + public abstract boolean processNonInlinedInvoke(CoreProviders providers, CallTargetNode node); } protected final NodePlugin[] nodePlugins; @@ -85,7 +89,7 @@ protected InlineBeforeAnalysisPolicy(NodePlugin[] nodePlugins) { this.nodePlugins = nodePlugins; } - protected abstract boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args); + protected abstract boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args); protected abstract InlineInfo createInvokeInfo(AnalysisMethod method); @@ -97,8 +101,7 @@ protected InlineBeforeAnalysisPolicy(NodePlugin[] nodePlugins) { protected abstract AbstractPolicyScope createRootScope(); - protected abstract AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle); + protected abstract AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method); /** @see InlineBeforeAnalysisGraphDecoder#shouldOmitIntermediateMethodInStates */ protected boolean shouldOmitIntermediateMethodInState(AnalysisMethod method) { @@ -108,7 +111,7 @@ protected boolean shouldOmitIntermediateMethodInState(AnalysisMethod method) { public static final InlineBeforeAnalysisPolicy NO_INLINING = new InlineBeforeAnalysisPolicy(new NodePlugin[0]) { @Override - protected boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args) { + protected boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args) { return false; } @@ -143,8 +146,7 @@ protected AbstractPolicyScope createRootScope() { } @Override - protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) { + protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method) { throw AnalysisError.shouldNotReachHere("NO_INLINING policy should not try to inline"); } }; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VarHandleFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VarHandleFeature.java index 9e74855377c7..c79167ebf497 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VarHandleFeature.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/VarHandleFeature.java @@ -27,6 +27,7 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.VarHandle; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.HashMap; import java.util.Map; @@ -99,6 +100,70 @@ public class VarHandleFeature implements InternalFeature { private final ConcurrentMap processedVarHandles = new ConcurrentHashMap<>(); private Consumer markAsUnsafeAccessed; + @Override + public void duringSetup(DuringSetupAccess access) { + /* + * Initialize fields of VarHandle instances that are @Stable eagerly, so that during method + * handle intrinsification loads of those fields and array elements can be constant-folded. + * + * Note that we do this on purpose here in an object replacer, and not in an object + * reachability handler: Intrinsification happens as part of method inlining before + * analysis, i.e., before the static analysis, i.e., before the VarHandle object itself is + * marked as reachable. The goal of intrinsification is to actually avoid making the + * VarHandle object itself reachable. + */ + access.registerObjectReplacer(VarHandleFeature::eagerlyInitializeVarHandle); + } + + private static Object eagerlyInitializeVarHandle(Object obj) { + if (obj instanceof VarHandle varHandle) { + eagerlyInitializeVarHandle(varHandle); + } + return obj; + } + + private static final Field varHandleVFormField = ReflectionUtil.lookupField(VarHandle.class, "vform"); + private static final Method varFormInitMethod = ReflectionUtil.lookupMethod(ReflectionUtil.lookupClass(false, "java.lang.invoke.VarForm"), "getMethodType_V", int.class); + private static final Method varHandleGetMethodHandleMethod = ReflectionUtil.lookupMethod(VarHandle.class, "getMethodHandle", int.class); + + public static void eagerlyInitializeVarHandle(VarHandle varHandle) { + try { + /* + * The field VarHandle.vform.methodType_V_table is a @Stable field but initialized + * lazily on first access. Therefore, constant folding can happen only after + * initialization has happened. We force initialization by invoking the method + * VarHandle.vform.getMethodType_V(0). + */ + Object varForm = varHandleVFormField.get(varHandle); + varFormInitMethod.invoke(varForm, 0); + + /* + * The AccessMode used for the access that we are going to intrinsify is hidden in a + * AccessDescriptor object that is also passed in as a parameter to the intrinsified + * method. Initializing all AccessMode enum values is easier than trying to extract the + * actual AccessMode. + */ + for (VarHandle.AccessMode accessMode : VarHandle.AccessMode.values()) { + /* + * Force initialization of the @Stable field VarHandle.vform.memberName_table. + */ + boolean isAccessModeSupported = varHandle.isAccessModeSupported(accessMode); + /* + * Force initialization of the @Stable field + * VarHandle.typesAndInvokers.methodType_table. + */ + varHandle.accessModeType(accessMode); + + if (isAccessModeSupported) { + /* Force initialization of the @Stable field VarHandle.methodHandleTable. */ + varHandleGetMethodHandleMethod.invoke(varHandle, accessMode.ordinal()); + } + } + } catch (ReflectiveOperationException ex) { + throw VMError.shouldNotReachHere(ex); + } + } + @Override public void afterRegistration(AfterRegistrationAccess access) { try { diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java index c230b49e1cde..772752493d6c 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/NativeImage.java @@ -1115,12 +1115,21 @@ private int completeImageBuild() { imageBuilderJavaArgs.add("-Djava.lang.invoke.InnerClassLambdaMetafactory.initializeLambdas=false"); /* * DONT_INLINE_THRESHOLD is used to set a profiling threshold for certain method handles and - * only allow inlining after n invocations. This is used for example in the implementation - * of record equals methods. We disable this behavior in the image builder because it can - * prevent optimizing the method handles for AOT compilation if the threshold is not - * reached. + * only allow inlining when JIT compiling after n invocations. PROFILE_GWT is used to + * profile "guard with test" method handles and speculate on a constant guard value, making + * the other branch statically unreachable for JIT compilation. + * + * Both are used for example in the implementation of record hashCode/equals methods. We + * disable this behavior in the image builder because for AOT compilation, profiling and + * speculation are never useful. Instead, it prevents optimizing the method handles for AOT + * compilation if the threshold is not already reached at image build time. + * + * As a side effect, the profiling is also disabled when using such method handles in the + * image generator itself. If that turns out to be a performance problem, we need to + * investigate at different solution that disables the profiling only for AOT compilation. */ imageBuilderJavaArgs.add("-Djava.lang.invoke.MethodHandle.DONT_INLINE_THRESHOLD=-1"); + imageBuilderJavaArgs.add("-Djava.lang.invoke.MethodHandle.PROFILE_GWT=false"); /* After JavaArgs consolidation add the user provided JavaArgs */ boolean afterOption = false; diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompilationFeature.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompilationFeature.java index 2ed14e40d39d..993f34f62eb1 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompilationFeature.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/hosted/runtimecompilation/RuntimeCompilationFeature.java @@ -28,7 +28,6 @@ import static com.oracle.svm.common.meta.MultiMethod.ORIGINAL_METHOD; import static com.oracle.svm.core.util.VMError.guarantee; import static com.oracle.svm.hosted.code.SubstrateCompilationDirectives.RUNTIME_COMPILED_METHOD; -import static com.oracle.svm.hosted.phases.InlineBeforeAnalysisPolicyUtils.Options.InlineBeforeAnalysisAllowedDepth; import static jdk.graal.compiler.java.BytecodeParserOptions.InlineDuringParsingMaxDepth; import java.lang.reflect.Executable; @@ -45,6 +44,7 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import org.graalvm.nativeimage.AnnotationAccess; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.hosted.Feature; @@ -799,7 +799,6 @@ public Function getStrengthenGraphsToTargetFunct * {@code RUNTIME_COMPILED_METHOD}s. */ private class RuntimeCompilationInlineBeforeAnalysisPolicy extends InlineBeforeAnalysisPolicy { - private final int accumulativeAllowedInliningDepth = InlineBeforeAnalysisAllowedDepth.getValue(); private final int trivialAllowingInliningDepth = InlineDuringParsingMaxDepth.getValue(HostedOptionValues.singleton()); final SVMHost hostVM; @@ -831,26 +830,22 @@ protected FixedWithNextNode processInvokeArgs(AnalysisMethod targetMethod, Fixed } @Override - protected boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args) { - if (inliningUtils.alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { - return true; - } - // worse case depth is max trivial, and then max accumulative - if (b.getDepth() > trivialAllowingInliningDepth + accumulativeAllowedInliningDepth) { - return false; - } - if (b.recursiveInliningDepth(method) > 0) { - /* Prevent recursive inlining. */ + protected boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args) { + if (allowInliningPredicate.allowInlining(b, method) != AllowInliningPredicate.InlineDecision.INLINE) { return false; } - if (!InlineBeforeAnalysisPolicyUtils.inliningAllowed(hostVM, b, method)) { - return false; + InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope accScope; + if (policyScope instanceof InlineBeforeAnalysisPolicyUtils.AlwaysInlineScope) { + /* + * If we are in "trivial inlining" mode, we make inlining decisions as if we are + * still the root (= null) accumulative inlining scope. + */ + accScope = null; + } else { + accScope = (InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope) policyScope; } - - AllowInliningPredicate.InlineDecision result = allowInliningPredicate.allowInlining(b, method); - - return result == AllowInliningPredicate.InlineDecision.INLINE; + return inliningUtils.shouldInlineInvoke(b, hostVM, accScope, method); } @Override @@ -871,26 +866,33 @@ protected AbstractPolicyScope createRootScope() { } @Override - protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) { + protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method) { if (outer instanceof InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope accOuter) { /* * once the accumulative policy is activated, then we cannot return to the trivial * policy */ - return inliningUtils.createAccumulativeInlineScope(accOuter, metaAccess, method, constArgsWithReceiver, intrinsifiedMethodHandle); + return inliningUtils.createAccumulativeInlineScope(accOuter, method); } assert outer == null || outer instanceof InlineBeforeAnalysisPolicyUtils.AlwaysInlineScope : "unexpected outer scope: " + outer; - // check if trivial is possible - boolean trivialInlineAllowed = hostVM.isAnalysisTrivialMethod(method); + /* + * Check if trivial is possible. We use the graph size as the main criteria, similar to + * the trivial inlining for AOT compilation. + * + * In addition, we do not allow method handle internals to be processed by the trivial + * inlining. The regular accumulative inlining scope has a special mode for method + * handle intrinsification with larger thresholds in order to fully inline the method + * handle. + */ + boolean trivialInlineAllowed = hostVM.isAnalysisTrivialMethod(method) && !AnnotationAccess.isAnnotationPresent(method, InlineBeforeAnalysisPolicyUtils.COMPILED_LAMBDA_FORM_ANNOTATION); int inliningDepth = outer == null ? 1 : outer.inliningDepth + 1; if (trivialInlineAllowed && inliningDepth <= trivialAllowingInliningDepth) { return new InlineBeforeAnalysisPolicyUtils.AlwaysInlineScope(inliningDepth); } else { // start with a new accumulative inline scope - return inliningUtils.createAccumulativeInlineScope(null, metaAccess, method, constArgsWithReceiver, intrinsifiedMethodHandle); + return inliningUtils.createAccumulativeInlineScope(null, method); } } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java index 13edc84ff9f7..2ef1b608213d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ModuleLayerFeature.java @@ -66,6 +66,7 @@ import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.jdk.Resources; import com.oracle.svm.core.jdk.RuntimeModuleSupport; import com.oracle.svm.core.util.VMError; @@ -139,6 +140,14 @@ public void duringSetup(DuringSetupAccess access) { private Object replaceHostedModules(Object source) { if (source instanceof Module module) { return moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(module); + } else if (source instanceof Class clazz) { + /* + * If the field Class(=DynamicHub).module is not reachable, we do not see all Module + * instances directly. So we also need to scan the module in Class/DynamicHub objects. + */ + moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(clazz.getModule()); + } else if (source instanceof DynamicHub hub) { + moduleLayerFeatureUtils.getOrCreateRuntimeModuleForHostedModule(hub.getModule()); } return source; } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index a9e4460375ef..6752dc122c6c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -670,6 +670,7 @@ protected void doRun(Map entryPoints, JavaMainSupport j /* Re-run shadow heap verification after compilation. */ aUniverse.getHeapVerifier().checkHeapSnapshot(debug, hMetaAccess, "after compilation", bb.getUniverse().getEmbeddedRoots()); + bb.getUnsupportedFeatures().report(bb); CodeCacheProvider codeCacheProvider = runtimeConfiguration.getBackendForNormalMethod().getProviders().getCodeCache(); reporter.printCreationStart(); @@ -751,6 +752,7 @@ private void verifyAndSealShadowHeap(NativeImageCodeCache codeCache, DebugContex */ Map embeddedConstants = codeCache.initAndGetEmbeddedConstants(); bb.getUniverse().getHeapVerifier().checkHeapSnapshot(debug, heap.hMetaAccess, "before heap layout", embeddedConstants); + bb.getUnsupportedFeatures().report(bb); /* * Seal shadow heap after final verification. Any modification to the shadow heap after this * point, i.e., registering a new ImageHeapConstant or materializing the hosted values diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/SimulateClassInitializerPolicy.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/SimulateClassInitializerPolicy.java index e697a824b451..69d32dc917e5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/SimulateClassInitializerPolicy.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/classinitialization/SimulateClassInitializerPolicy.java @@ -38,11 +38,13 @@ import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.graph.NodeSourcePosition; +import jdk.graal.compiler.nodes.CallTargetNode; import jdk.graal.compiler.nodes.FixedWithNextNode; import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin; +import jdk.graal.compiler.nodes.spi.CoreProviders; /** * This class is necessary because simulation of class initializer is based on @@ -91,6 +93,11 @@ public boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, return true; } + @Override + public boolean processNonInlinedInvoke(CoreProviders providers, CallTargetNode node) { + return true; + } + @Override public String toString() { return "allocatedBytes: " + allocatedBytes + " (" + accumulativeCounters.allocatedBytes + ")"; @@ -108,7 +115,7 @@ public String toString() { } @Override - protected boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args) { + protected boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args) { if (b.getDepth() > support.maxInlineDepth) { /* Safeguard against excessive inlining, for example endless recursion. */ return false; @@ -154,8 +161,7 @@ protected AbstractPolicyScope createRootScope() { } @Override - protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope o, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) { + protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope o, AnalysisMethod method) { var outer = (SimulateClassInitializerInlineScope) o; return new SimulateClassInitializerInlineScope(outer.accumulativeCounters, outer.inliningDepth + 1); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java index 85f5563d6b47..a85ac193cfef 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyImpl.java @@ -24,11 +24,11 @@ */ package com.oracle.svm.hosted.phases; -import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; import com.oracle.graal.pointsto.meta.AnalysisMethod; import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy; import com.oracle.svm.core.ParsingReason; import com.oracle.svm.hosted.SVMHost; +import com.oracle.svm.hosted.phases.InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope; import jdk.graal.compiler.graph.NodeSourcePosition; import jdk.graal.compiler.nodes.FixedWithNextNode; @@ -37,23 +37,7 @@ import jdk.graal.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; import jdk.graal.compiler.nodes.graphbuilderconf.NodePlugin; -/** - * The defaults for node limits are very conservative. Only small methods should be inlined. The - * only exception are constants - an arbitrary number of constants is always allowed. Limiting to 1 - * node (which can be also 1 invoke) means that field accessors can be inlined and forwarding - * methods can be inlined. But null checks and class initialization checks are already putting a - * method above the limit. On the other hand, the inlining depth is generous because we do do not - * need to limit it. Note that more experimentation is necessary to come up with the optimal - * configuration. - * - * Important: the implementation details of this class are publicly observable API. Since - * {@link java.lang.reflect.Method} constants can be produced by inlining lookup methods with - * constant arguments, reducing inlining can break customer code. This means we can never reduce the - * amount of inlining in a future version without breaking compatibility. This also means that we - * must be conservative and only inline what is necessary for known use cases. - */ public class InlineBeforeAnalysisPolicyImpl extends InlineBeforeAnalysisPolicy { - private final int maxInliningDepth = InlineBeforeAnalysisPolicyUtils.Options.InlineBeforeAnalysisAllowedDepth.getValue(); private final SVMHost hostVM; private final InlineBeforeAnalysisPolicyUtils inliningUtils; @@ -65,19 +49,8 @@ public InlineBeforeAnalysisPolicyImpl(SVMHost hostVM, InlineBeforeAnalysisPolicy } @Override - protected boolean shouldInlineInvoke(GraphBuilderContext b, AnalysisMethod method, ValueNode[] args) { - if (inliningUtils.alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { - return true; - } - if (b.getDepth() >= maxInliningDepth) { - return false; - } - if (b.recursiveInliningDepth(method) > 0) { - /* Prevent recursive inlining. */ - return false; - } - - return InlineBeforeAnalysisPolicyUtils.inliningAllowed(hostVM, b, method); + protected boolean shouldInlineInvoke(GraphBuilderContext b, AbstractPolicyScope policyScope, AnalysisMethod method, ValueNode[] args) { + return inliningUtils.shouldInlineInvoke(b, hostVM, (AccumulativeInlineScope) policyScope, method); } @Override @@ -106,9 +79,8 @@ protected AbstractPolicyScope createRootScope() { } @Override - protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) { - return inliningUtils.createAccumulativeInlineScope((InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope) outer, metaAccess, method, constArgsWithReceiver, intrinsifiedMethodHandle); + protected AbstractPolicyScope openCalleeScope(AbstractPolicyScope outer, AnalysisMethod method) { + return inliningUtils.createAccumulativeInlineScope((InlineBeforeAnalysisPolicyUtils.AccumulativeInlineScope) outer, method); } @Override diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java index 3fe948fee521..9a8df7d0428d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/InlineBeforeAnalysisPolicyUtils.java @@ -25,21 +25,16 @@ package com.oracle.svm.hosted.phases; import java.lang.annotation.Annotation; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.util.Map; +import java.lang.reflect.Executable; import java.util.Set; import org.graalvm.nativeimage.AnnotationAccess; -import com.oracle.graal.pointsto.infrastructure.OriginalClassProvider; import com.oracle.graal.pointsto.meta.AnalysisMetaAccess; import com.oracle.graal.pointsto.meta.AnalysisMethod; -import com.oracle.graal.pointsto.meta.AnalysisType; import com.oracle.graal.pointsto.phases.InlineBeforeAnalysisPolicy; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.RestrictHeapAccess; -import com.oracle.svm.core.jdk.VarHandleFeature; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.ReachabilityRegistrationNode; @@ -62,10 +57,12 @@ import jdk.graal.compiler.nodes.StartNode; import jdk.graal.compiler.nodes.UnwindNode; import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.calc.ConditionalNode; import jdk.graal.compiler.nodes.extended.ValueAnchorNode; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.java.AbstractNewObjectNode; import jdk.graal.compiler.nodes.java.NewArrayNode; +import jdk.graal.compiler.nodes.spi.CoreProviders; import jdk.graal.compiler.nodes.spi.ValueProxy; import jdk.graal.compiler.nodes.virtual.AllocatedObjectNode; import jdk.graal.compiler.nodes.virtual.CommitAllocationNode; @@ -73,7 +70,23 @@ import jdk.graal.compiler.nodes.virtual.VirtualObjectNode; import jdk.graal.compiler.options.Option; import jdk.graal.compiler.replacements.nodes.MethodHandleWithExceptionNode; - +import jdk.internal.vm.annotation.ForceInline; + +/** + * The defaults for node limits are very conservative. Only small methods should be inlined. The + * only exception are constants - an arbitrary number of constants is always allowed. Limiting to 1 + * node (which can be also 1 invoke) means that field accessors can be inlined and forwarding + * methods can be inlined. But null checks and class initialization checks are already putting a + * method above the limit. On the other hand, the inlining depth is generous because we do do not + * need to limit it. Note that more experimentation is necessary to come up with the optimal + * configuration. + * + * Important: the implementation details of this class are publicly observable API. Since + * {@link java.lang.reflect.Method} constants can be produced by inlining lookup methods with + * constant arguments, reducing inlining can break customer code. This means we can never reduce the + * amount of inlining in a future version without breaking compatibility. This also means that we + * must be conservative and only inline what is necessary for known use cases. + */ public class InlineBeforeAnalysisPolicyUtils { public static class Options { @Option(help = "Maximum number of computation nodes for method inlined before static analysis")// @@ -85,26 +98,100 @@ public static class Options { @Option(help = "Maximum call depth for method inlined before static analysis")// public static final HostedOptionKey InlineBeforeAnalysisAllowedDepth = new HostedOptionKey<>(20); + @Option(help = "Maximum number of methods inlined for method inlined before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisAllowedInlinings = new HostedOptionKey<>(10_000); + @Option(help = "Maximum number of computation nodes for method handle internals inlined before static analysis")// - public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedNodes = new HostedOptionKey<>(100); + public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedNodes = new HostedOptionKey<>(10_000); @Option(help = "Maximum number of invokes for method handle internals inlined before static analysis")// - public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedInvokes = new HostedOptionKey<>(20); + public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedInvokes = new HostedOptionKey<>(1_000); + + @Option(help = "Maximum call depth for method handle internals inlined before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedDepth = new HostedOptionKey<>(1_000); + + @Option(help = "Maximum number of methods inlined for method handle internals before static analysis")// + public static final HostedOptionKey InlineBeforeAnalysisMethodHandleAllowedInlinings = new HostedOptionKey<>(10_000); } + /* Cached values of options, to avoid repeated option lookup. */ + public final int optionAllowedNodes = Options.InlineBeforeAnalysisAllowedNodes.getValue(); + public final int optionAllowedInvokes = Options.InlineBeforeAnalysisAllowedInvokes.getValue(); + public final int optionAllowedDepth = Options.InlineBeforeAnalysisAllowedDepth.getValue(); + public final int optionAllowedInlinings = Options.InlineBeforeAnalysisAllowedInlinings.getValue(); + public final int optionMethodHandleAllowedNodes = Options.InlineBeforeAnalysisMethodHandleAllowedNodes.getValue(); + public final int optionMethodHandleAllowedInvokes = Options.InlineBeforeAnalysisMethodHandleAllowedInvokes.getValue(); + public final int optionMethodHandleAllowedDepth = Options.InlineBeforeAnalysisMethodHandleAllowedDepth.getValue(); + public final int optionMethodHandleAllowedInlinings = Options.InlineBeforeAnalysisMethodHandleAllowedInlinings.getValue(); + @SuppressWarnings("unchecked") // - private static final Class COMPILED_LAMBDA_FORM_ANNOTATION = // + public static final Class COMPILED_LAMBDA_FORM_ANNOTATION = // (Class) ReflectionUtil.lookupClass(false, "java.lang.invoke.LambdaForm$Compiled"); - private static final Class INVOKERS_CLASS = ReflectionUtil.lookupClass(false, "java.lang.invoke.Invokers"); + public boolean shouldInlineInvoke(GraphBuilderContext b, SVMHost hostVM, AccumulativeInlineScope policyScope, AnalysisMethod method) { + boolean result = shouldInlineInvoke0(b, hostVM, policyScope, method); + if (result && policyScope != null) { + policyScope.accumulativeCounters.totalInlinedMethods++; + } + return result; + } + + private boolean shouldInlineInvoke0(GraphBuilderContext b, SVMHost hostVM, AccumulativeInlineScope policyScope, AnalysisMethod method) { + if (!inliningAllowed(hostVM, b, method)) { + /* + * Inlining this method can lead to incorrect code, so this condition must always be + * checked first. + */ + return false; + } else if (alwaysInlineInvoke((AnalysisMetaAccess) b.getMetaAccess(), method)) { + /* Manual override of the regular inlining depth checks. */ + return true; + } + + boolean inMethodHandleIntrinsification = policyScope != null && policyScope.accumulativeCounters.inMethodHandleIntrinsification; + int allowedInlinings = inMethodHandleIntrinsification ? optionMethodHandleAllowedInlinings : optionAllowedInlinings; + if (policyScope != null && policyScope.accumulativeCounters.totalInlinedMethods >= allowedInlinings) { + return false; + } - private static final Map, Set> IGNORED_METHOD_HANDLE_METHODS = Map.of( - MethodHandle.class, Set.of("bindTo"), - MethodHandles.class, Set.of("dropArguments", "filterReturnValue", "foldArguments", "insertArguments"), - INVOKERS_CLASS, Set.of("spreadInvoker")); + int allowedDepth = inMethodHandleIntrinsification ? optionMethodHandleAllowedDepth : optionAllowedDepth; + /* + * Note that we do not use the inlining depth from the GraphBuilderContext: If we are in a + * regular inlining scope, but nested into a deep method handle intrinsification, then the + * total inlining depth is high but our local depth for the scope can still be low enough to + * do inlining. + */ + int actualDepth = policyScope == null ? 0 : policyScope.inliningDepth; + if (actualDepth >= allowedDepth) { + return false; + } - private AnalysisType methodHandleType; - private AnalysisType varHandleGuardsType; + if (!inMethodHandleIntrinsification) { + if (b.recursiveInliningDepth(method) > 0) { + /* + * There is no need to make recursive inlining configurable via an option for now, + * it is always just disallowed. Except when we are in a method handle + * intrinsification, because method handles are often deeply recursive. The + * "total inlined methods" check above prevents excessive recursive inlining during + * method handle intrinsification. + */ + return false; + } + if (policyScope != null && AnnotationAccess.isAnnotationPresent(method, COMPILED_LAMBDA_FORM_ANNOTATION)) { + /* + * We are not in a method handle intrinsification i.e., the method handle was not + * the root inlining context. Do not inline compiled lambda forms at all. Since the + * inlining limits are low, this could lead to partially inlined method handle + * chains, which leads to slow code because the non-inlined part is executed in the + * method handle interpreter. It is likely that the method handle is fully inlined + * when processing the callee, where the method handle is then the root inlining + * context. + */ + return false; + } + } + return true; + } public static boolean inliningAllowed(SVMHost hostVM, GraphBuilderContext b, AnalysisMethod callee) { AnalysisMethod caller = (AnalysisMethod) b.getMethod(); @@ -175,6 +262,11 @@ public boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, return true; } + @Override + public boolean processNonInlinedInvoke(CoreProviders providers, CallTargetNode node) { + return true; + } + @Override public String toString() { return "AlwaysInlineScope"; @@ -182,33 +274,17 @@ public String toString() { } static final class AccumulativeCounters { - static AccumulativeCounters create(AccumulativeCounters outer) { - int maxDepth = (outer != null) ? outer.maxInliningDepth : Options.InlineBeforeAnalysisAllowedDepth.getValue(); - return new AccumulativeCounters(Options.InlineBeforeAnalysisAllowedNodes.getValue(), - Options.InlineBeforeAnalysisAllowedInvokes.getValue(), - maxDepth, - false); - } - - static AccumulativeCounters createForMethodHandleIntrinsification(AccumulativeCounters outer) { - return new AccumulativeCounters(Options.InlineBeforeAnalysisMethodHandleAllowedNodes.getValue(), - Options.InlineBeforeAnalysisMethodHandleAllowedInvokes.getValue(), - outer.maxInliningDepth, - true); - } - int maxNodes; int maxInvokes; - final int maxInliningDepth; final boolean inMethodHandleIntrinsification; - int numNodes = 0; - int numInvokes = 0; + int numNodes; + int numInvokes; + int totalInlinedMethods; - private AccumulativeCounters(int maxNodes, int maxInvokes, int maxInliningDepth, boolean inMethodHandleIntrinsification) { + private AccumulativeCounters(int maxNodes, int maxInvokes, boolean inMethodHandleIntrinsification) { this.maxNodes = maxNodes; this.maxInvokes = maxInvokes; - this.maxInliningDepth = maxInliningDepth; this.inMethodHandleIntrinsification = inMethodHandleIntrinsification; } } @@ -218,8 +294,7 @@ private AccumulativeCounters(int maxNodes, int maxInvokes, int maxInliningDepth, * has exceeded a specified count, or an illegal node is inlined, then the process will be * aborted. */ - public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineScope outer, AnalysisMetaAccess metaAccess, - AnalysisMethod method, boolean[] constArgsWithReceiver, boolean intrinsifiedMethodHandle) { + public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineScope outer, AnalysisMethod method) { AccumulativeCounters accumulativeCounters; int depth; if (outer == null) { @@ -228,16 +303,17 @@ public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineS * point of view. */ depth = 1; - accumulativeCounters = AccumulativeCounters.create(null); - } else if (!outer.accumulativeCounters.inMethodHandleIntrinsification && (intrinsifiedMethodHandle || isMethodHandleIntrinsificationRoot(metaAccess, method, constArgsWithReceiver))) { - /* - * Method handle intrinsification root: create counters with relaxed limits and permit - * more types of nodes, but not recursively, i.e., not if we are already in a method - * handle intrinsification context. - */ - depth = outer.inliningDepth + 1; - accumulativeCounters = AccumulativeCounters.createForMethodHandleIntrinsification(outer.accumulativeCounters); + if (AnnotationAccess.isAnnotationPresent(method, COMPILED_LAMBDA_FORM_ANNOTATION)) { + /* + * Method handle intrinsification root: create counters with relaxed limits and + * permit more types of nodes, but not recursively, i.e., not if we are already in a + * method handle intrinsification context. + */ + accumulativeCounters = new AccumulativeCounters(optionMethodHandleAllowedNodes, optionMethodHandleAllowedInvokes, true); + } else { + accumulativeCounters = new AccumulativeCounters(optionAllowedNodes, optionAllowedInvokes, false); + } } else if (outer.accumulativeCounters.inMethodHandleIntrinsification && !inlineForMethodHandleIntrinsification(method)) { /* @@ -249,9 +325,12 @@ public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineS * This assumes that the regular limits are strict enough to prevent excessive inlining * triggered by method handles. We could also use alternative fixed values or the option * defaults instead of any set option values. + * + * We start again with an inlining depth of 1, i.e., we behave as if that method is the + * inlining root. */ - depth = outer.inliningDepth + 1; - accumulativeCounters = AccumulativeCounters.create(outer.accumulativeCounters); + depth = 1; + accumulativeCounters = new AccumulativeCounters(optionAllowedNodes, optionAllowedInvokes, false); } else { /* Nested inlining (potentially during method handle intrinsification). */ @@ -261,54 +340,6 @@ public AccumulativeInlineScope createAccumulativeInlineScope(AccumulativeInlineS return new AccumulativeInlineScope(accumulativeCounters, depth); } - private boolean isMethodHandleIntrinsificationRoot(AnalysisMetaAccess metaAccess, AnalysisMethod method, boolean[] constArgsWithReceiver) { - return (isVarHandleMethod(metaAccess, method) || hasConstantMethodHandleParameter(metaAccess, method, constArgsWithReceiver)) && !isIgnoredMethodHandleMethod(method); - } - - private boolean hasConstantMethodHandleParameter(AnalysisMetaAccess metaAccess, AnalysisMethod method, boolean[] constArgsWithReceiver) { - if (methodHandleType == null) { - methodHandleType = metaAccess.lookupJavaType(MethodHandle.class); - } - for (int i = 0; i < constArgsWithReceiver.length; i++) { - if (constArgsWithReceiver[i] && methodHandleType.isAssignableFrom(getParameterType(method, i))) { - return true; - } - } - return false; - } - - private static AnalysisType getParameterType(AnalysisMethod method, int index) { - int i = index; - if (!method.isStatic()) { - if (i == 0) { // receiver - return method.getDeclaringClass(); - } - i--; - } - return method.getSignature().getParameterType(i); - } - - /** - * Checks if the method is the intrinsification root for a VarHandle. In the current VarHandle - * implementation, all guards are in the automatically generated class VarHandleGuards. All - * methods do have a VarHandle argument, and we expect it to be a compile-time constant. - *

- * See the documentation in {@link VarHandleFeature} for more information on the overall - * VarHandle support. - */ - private boolean isVarHandleMethod(AnalysisMetaAccess metaAccess, AnalysisMethod method) { - if (varHandleGuardsType == null) { - varHandleGuardsType = metaAccess.lookupJavaType(ReflectionUtil.lookupClass(false, "java.lang.invoke.VarHandleGuards")); - } - return method.getDeclaringClass().equals(varHandleGuardsType); - } - - private static boolean isIgnoredMethodHandleMethod(AnalysisMethod method) { - Class declaringClass = OriginalClassProvider.getJavaClass(method.getDeclaringClass()); - Set ignoredMethods = IGNORED_METHOD_HANDLE_METHODS.get(declaringClass); - return ignoredMethods != null && ignoredMethods.contains(method.getName()); - } - public final class AccumulativeInlineScope extends InlineBeforeAnalysisPolicy.AbstractPolicyScope { final AccumulativeCounters accumulativeCounters; @@ -374,11 +405,6 @@ public boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, return true; } - if (inliningDepth > accumulativeCounters.maxInliningDepth) { - // too deep to continue inlining - return false; - } - if (node instanceof FullInfopointNode || node instanceof ValueProxy || node instanceof ValueAnchorNode || node instanceof FrameState || node instanceof AbstractBeginNode || node instanceof AbstractEndNode) { /* @@ -395,6 +421,14 @@ public boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, return true; } + if (node instanceof ConditionalNode) { + /* + * A ConditionalNode is used to "materialize" a prior logic node when returning a + * condition in a boolean method. We do not want to count it separately. + */ + return true; + } + if (node instanceof ReachabilityRegistrationNode) { /* * These nodes do not affect compilation and are only used to execute handlers @@ -462,25 +496,43 @@ public boolean processNode(AnalysisMetaAccess metaAccess, AnalysisMethod method, return allow || accumulativeCounters.inMethodHandleIntrinsification; } + @Override + public boolean processNonInlinedInvoke(CoreProviders providers, CallTargetNode node) { + if (node.targetMethod() != null && AnnotationAccess.isAnnotationPresent(node.targetMethod(), COMPILED_LAMBDA_FORM_ANNOTATION)) { + /* + * Prevent partial inlining of method handle code, both with and without + * "intrinsification of method handles". We rather want to keep a top-level call to + * the original method handle. + */ + return false; + } + return true; + } + @Override public String toString() { return "AccumulativeInlineScope: " + numNodes + "/" + numInvokes + " (" + accumulativeCounters.numNodes + "/" + accumulativeCounters.numInvokes + ")"; } } + private static final Set> INLINE_METHOD_HANDLE_CLASSES = Set.of( + /* Inline trivial helper methods for value conversion. */ + sun.invoke.util.ValueConversions.class); + + private static final Set INLINE_METHOD_HANDLE_METHODS = Set.of( + /* + * Important methods in the method handle implementation that do not have + * a @ForceInline annotation. + */ + ReflectionUtil.lookupMethod(ReflectionUtil.lookupClass(false, "java.lang.invoke.DirectMethodHandle"), "allocateInstance", Object.class), + ReflectionUtil.lookupMethod(ReflectionUtil.lookupClass(false, "java.lang.invoke.DirectMethodHandle$Accessor"), "checkCast", Object.class), + ReflectionUtil.lookupMethod(ReflectionUtil.lookupClass(false, "java.lang.invoke.DirectMethodHandle$StaticAccessor"), "checkCast", Object.class)); + private static boolean inlineForMethodHandleIntrinsification(AnalysisMethod method) { - String className = method.getDeclaringClass().toJavaName(true); - if (className.startsWith("java.lang.invoke") && !className.contains("InvokerBytecodeGenerator")) { - /* - * Inline all helper methods used by method handles. We do not know exactly which ones - * they are, but they are all from the same package. - */ - return true; - } else if (className.equals("sun.invoke.util.ValueConversions")) { - /* Inline trivial helper methods for value conversion. */ - return true; - } - return false; + return AnnotationAccess.isAnnotationPresent(method, ForceInline.class) || + AnnotationAccess.isAnnotationPresent(method, COMPILED_LAMBDA_FORM_ANNOTATION) || + INLINE_METHOD_HANDLE_CLASSES.contains(method.getDeclaringClass().getJavaClass()) || + INLINE_METHOD_HANDLE_METHODS.contains(method.getJavaMethod()); } /** diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/IntrinsifyMethodHandlesInvocationPlugin.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/IntrinsifyMethodHandlesInvocationPlugin.java index 242acbc508eb..be055c089ecc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/IntrinsifyMethodHandlesInvocationPlugin.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/IntrinsifyMethodHandlesInvocationPlugin.java @@ -30,8 +30,6 @@ import java.lang.invoke.MethodType; import java.lang.invoke.VarHandle; import java.lang.invoke.WrongMethodTypeException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -59,7 +57,6 @@ import com.oracle.svm.hosted.meta.HostedSnippetReflectionProvider; import com.oracle.svm.hosted.meta.HostedType; import com.oracle.svm.hosted.meta.HostedUniverse; -import com.oracle.svm.util.ReflectionUtil; import jdk.graal.compiler.api.replacements.SnippetReflectionProvider; import jdk.graal.compiler.core.common.spi.MetaAccessExtensionProvider; @@ -175,21 +172,6 @@ */ public class IntrinsifyMethodHandlesInvocationPlugin implements NodePlugin { - private static final Field varHandleVFormField; - private static final Method varFormInitMethod; - private static final Method varHandleGetMethodHandleMethod; - - static { - varHandleVFormField = ReflectionUtil.lookupField(VarHandle.class, "vform"); - try { - Class varFormClass = Class.forName("java.lang.invoke.VarForm"); - varFormInitMethod = ReflectionUtil.lookupMethod(varFormClass, "getMethodType_V", int.class); - varHandleGetMethodHandleMethod = ReflectionUtil.lookupMethod(VarHandle.class, "getMethodHandle", int.class); - } catch (ClassNotFoundException ex) { - throw VMError.shouldNotReachHere(ex); - } - } - private final ParsingReason reason; private final Providers parsingProviders; private final HostedProviders universeProviders; @@ -317,48 +299,7 @@ private boolean isVarHandleMethod(ResolvedJavaMethod method, ValueNode[] args) { if (args.length < 1 || !args[0].isJavaConstant() || !isVarHandle(args[0])) { return false; } - - try { - /* - * The field VarHandle.vform.methodType_V_table is a @Stable field but initialized - * lazily on first access. Therefore, constant folding can happen only after - * initialization has happened. We force initialization by invoking the method - * VarHandle.vform.getMethodType_V(0). - */ - VarHandle varHandle = hostedSnippetReflection.asObject(VarHandle.class, args[0].asJavaConstant()); - Object varForm = varHandleVFormField.get(varHandle); - varFormInitMethod.invoke(varForm, 0); - - /* - * The AccessMode used for the access that we are going to intrinsify is hidden in a - * AccessDescriptor object that is also passed in as a parameter to the intrinsified - * method. Initializing all AccessMode enum values is easier than trying to extract - * the actual AccessMode. - */ - for (VarHandle.AccessMode accessMode : VarHandle.AccessMode.values()) { - /* - * Force initialization of the @Stable field VarHandle.vform.memberName_table. - * Starting with JDK 17, this field is lazily initialized. - */ - boolean isAccessModeSupported = varHandle.isAccessModeSupported(accessMode); - /* - * Force initialization of the @Stable field - * VarHandle.typesAndInvokers.methodType_table. - */ - varHandle.accessModeType(accessMode); - - if (isAccessModeSupported) { - /* - * Force initialization of the @Stable field VarHandle.methodHandleTable (or - * VarHandle.typesAndInvokers.methodHandle_tabel on JDK <= 17) . - */ - varHandleGetMethodHandleMethod.invoke(varHandle, accessMode.ordinal()); - } - } - } catch (ReflectiveOperationException ex) { - throw VMError.shouldNotReachHere(ex); - } - + VarHandleFeature.eagerlyInitializeVarHandle(hostedSnippetReflection.asObject(VarHandle.class, args[0].asJavaConstant())); return true; } else { return false; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/ReflectionPlugins.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/ReflectionPlugins.java index 719033927eb5..b7a0ed3474a2 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/ReflectionPlugins.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/snippets/ReflectionPlugins.java @@ -635,12 +635,21 @@ private Object unboxObjectConstant(GraphBuilderContext b, JavaConstant argConsta /* Any other object that is not a Class. */ Object result = snippetReflection.asObject(Object.class, argConstant); - if (result != null && ALLOWED_CONSTANT_CLASSES.contains(result.getClass())) { + if (result != null && isAllowedConstant(result.getClass())) { return result; } return null; } + private boolean isAllowedConstant(Class clazz) { + for (var allowed : ALLOWED_CONSTANT_CLASSES) { + if (allowed.isAssignableFrom(clazz)) { + return true; + } + } + return false; + } + /** * This method checks if the element should be intrinsified and returns the cached intrinsic * element if found. Caching intrinsic elements during analysis and reusing the same element