JIT: Inlining via splitting

src/coreclr/jit/async.cpp

@@ -173,7 +173,8 @@ PhaseStatus Compiler::SaveAsyncContexts()
         CORINFO_CALL_INFO callInfo = {};
         callInfo.hMethod           = captureCall->gtCallMethHnd;
         callInfo.methodFlags       = info.compCompHnd->getMethodAttribs(callInfo.hMethod);
-        impMarkInlineCandidate(captureCall, MAKE_METHODCONTEXT(callInfo.hMethod), false, &callInfo, compInlineContext);
+        impMarkInlineCandidate(captureCall, MAKE_METHODCONTEXT(callInfo.hMethod), false, &callInfo, compInlineContext,
+                               DebugInfo());
 
         Statement* captureStmt = fgNewStmtFromTree(captureCall);
         fgInsertStmtAtBeg(fgFirstBB, captureStmt);
@@ -372,7 +373,8 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl)
     CORINFO_CALL_INFO callInfo = {};
     callInfo.hMethod           = restoreCall->gtCallMethHnd;
     callInfo.methodFlags       = info.compCompHnd->getMethodAttribs(callInfo.hMethod);
-    impMarkInlineCandidate(restoreCall, MAKE_METHODCONTEXT(callInfo.hMethod), false, &callInfo, compInlineContext);
+    impMarkInlineCandidate(restoreCall, MAKE_METHODCONTEXT(callInfo.hMethod), false, &callInfo, compInlineContext,
+                           DebugInfo());
 
     Statement* restoreStmt = fgNewStmtFromTree(restoreCall);
     fgInsertStmtAtEnd(newReturnBB, restoreStmt);

src/coreclr/jit/compiler.h

@@ -2559,6 +2559,7 @@ class Compiler
     friend class MorphCopyBlockHelper;
     friend class SharedTempsScope;
     friend class CallArgs;
+    friend class InlineAndDevirtualizeWalker;
     friend class IndirectCallTransformer;
     friend class ProfileSynthesis;
     friend class LocalsUseVisitor;
@@ -3467,7 +3468,6 @@ class Compiler
     GenTree* gtNewMustThrowException(unsigned helper, var_types type, CORINFO_CLASS_HANDLE clsHnd);
 
     GenTreeLclFld* gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset, ClassLayout* layout = nullptr);
-    GenTreeRetExpr* gtNewInlineCandidateReturnExpr(GenTreeCall* inlineCandidate, var_types type);
 
     GenTreeFieldAddr* gtNewFieldAddrNode(var_types            type,
                                          CORINFO_FIELD_HANDLE fldHnd,
@@ -3679,7 +3679,7 @@ class Compiler
                                    bool         ignoreRoot       = false);
 
     bool gtSplitTree(
-        BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse, bool early = false);
+        BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitPointUse, bool includeOperands = true, bool early = false);
 
     bool gtStoreMayDefineField(
         LclVarDsc* fieldVarDsc, ssize_t offset, ValueSize size, ssize_t* pFieldRelativeOffset, ValueSize* pFieldAffectedBytes);
@@ -5156,10 +5156,11 @@ class Compiler
                            unsigned               methAttr,
                            CORINFO_CONTEXT_HANDLE exactContextHnd,
                            InlineContext*         inlinersContext,
+                           const DebugInfo&       debugInfo,
                            InlineCandidateInfo**  ppInlineCandidateInfo,
                            InlineResult*          inlineResult);
 
-    void impInlineRecordArgInfo(InlineInfo* pInlineInfo, CallArg* arg, InlArgInfo* argInfo, InlineResult* inlineResult);
+    void impInlineRecordArgInfo(InlineInfo* pInlineInfo, InlArgInfo* argInfo, CallArg* arg, InlineResult* inlineResult);
 
     void impInlineInitVars(InlineInfo* pInlineInfo);
 
@@ -5178,15 +5179,17 @@ class Compiler
                                 CORINFO_CONTEXT_HANDLE exactContextHnd,
                                 bool                   exactContextNeedsRuntimeLookup,
                                 CORINFO_CALL_INFO*     callInfo,
-                                InlineContext*         inlinersContext);
+                                InlineContext*         inlinersContext,
+                                const DebugInfo&       debugInfo);
 
     void impMarkInlineCandidateHelper(GenTreeCall*           call,
                                       uint8_t                candidateIndex,
                                       CORINFO_CONTEXT_HANDLE exactContextHnd,
                                       bool                   exactContextNeedsRuntimeLookup,
                                       CORINFO_CALL_INFO*     callInfo,
                                       InlineContext*         inlinersContext,
-                                      InlineResult*          inlineResult);
+                                      InlineResult*          inlineResult,
+                                      const DebugInfo&       debugInfo);
 
     bool impTailCallRetTypeCompatible(bool                     allowWidening,
                                       var_types                callerRetType,
@@ -5555,6 +5558,7 @@ class Compiler
     BasicBlock* fgSplitBlockAtBeginning(BasicBlock* curr);
     BasicBlock* fgSplitBlockAtEnd(BasicBlock* curr);
     BasicBlock* fgSplitBlockAfterStatement(BasicBlock* curr, Statement* stmt);
+    BasicBlock* fgSplitBlockBeforeStatement(BasicBlock* curr, Statement* stmt);
     BasicBlock* fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node); // for LIR
     BasicBlock* fgSplitEdge(BasicBlock* curr, BasicBlock* succ);
     BasicBlock* fgSplitBlockBeforeTree(BasicBlock* block, Statement* stmt, GenTree* splitPoint, Statement** firstNewStmt, GenTree*** splitNodeUse);
@@ -6472,6 +6476,9 @@ class Compiler
 #endif
 
 public:
+    //                  Create a new temporary variable to hold the result of *ppTree,
+    //                  and transform the graph accordingly.
+    GenTree* fgInsertCommaFormTemp(GenTree** ppTree);
     Statement* fgNewStmtAtBeg(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
     void fgInsertStmtAtEnd(BasicBlock* block, Statement* stmt);
     Statement* fgNewStmtAtEnd(BasicBlock* block, GenTree* tree, const DebugInfo& di = DebugInfo());
@@ -6483,9 +6490,9 @@ class Compiler
     void fgInsertStmtAfter(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
     void fgInsertStmtBefore(BasicBlock* block, Statement* insertionPoint, Statement* stmt);
 
-    //                  Create a new temporary variable to hold the result of *ppTree,
-    //                  and transform the graph accordingly.
-    GenTree* fgInsertCommaFormTemp(GenTree** ppTree);
+private:
+    void fgInsertStmtListAtEnd(BasicBlock* block, Statement* stmtList);
+    void fgInsertStmtListBefore(BasicBlock* block, Statement* stmtBefore, Statement* stmtList);
 
 private:
     Statement* fgInsertStmtListAfter(BasicBlock* block, Statement* stmtAfter, Statement* stmtList);
@@ -6626,10 +6633,10 @@ class Compiler
     GenTree* fgMorphCall(GenTreeCall* call);
     GenTree* fgExpandVirtualVtableCallTarget(GenTreeCall* call);
 
-    void fgMorphCallInline(GenTreeCall* call, InlineResult* result);
-    void fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
+    void fgMorphCallInline(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result);
+    void fgMorphCallInlineHelper(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
 #if DEBUG
-    void fgNoteNonInlineCandidate(Statement* stmt, GenTreeCall* call);
+    void fgNoteNonInlineCandidate(GenTreeCall* call);
     static fgWalkPreFn fgFindNonInlineCandidate;
 #endif
     GenTree* fgOptimizeDelegateConstructor(GenTreeCall*            call,
@@ -6808,11 +6815,11 @@ class Compiler
     bool MethodInstantiationComplexityExceeds(CORINFO_METHOD_HANDLE handle, int& cur, int max);
     bool TypeInstantiationComplexityExceeds(CORINFO_CLASS_HANDLE handle, int& cur, int max);
 
-    void fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
-    void fgInsertInlineeBlocks(InlineInfo* pInlineInfo);
-    void fgInsertInlineeArgument(const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI);
-    Statement* fgInlinePrependStatements(InlineInfo* inlineInfo);
-    void fgInlineAppendStatements(InlineInfo* inlineInfo, BasicBlock* block, Statement* stmt);
+    void fgInvokeInlineeCompiler(InlineInfo& inlineInfo, GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
+    void fgFinalizeInlineeStatements(InlineInfo* pInlineInfo);
+    void fgInsertInlineeArgument(class StatementListBuilder& statements, const InlArgInfo& argInfo, const DebugInfo& callDI);
+    void fgInlinePrependStatements(InlineInfo* inlineInfo);
+    void fgInlineAppendStatements(class StatementListBuilder& statements, InlineInfo* inlineInfo);
 
 #ifdef DEBUG
     static fgWalkPreFn fgDebugCheckInlineCandidates;
@@ -12156,7 +12163,6 @@ class GenTreeVisitor
             case GT_ASYNC_RESUME_INFO:
             case GT_LABEL:
             case GT_FTN_ADDR:
-            case GT_RET_EXPR:
             case GT_CNS_INT:
             case GT_CNS_LNG:
             case GT_CNS_DBL:

src/coreclr/jit/compiler.hpp

@@ -4360,7 +4360,6 @@ GenTree::VisitResult GenTree::VisitOperands(TVisitor visitor)
         case GT_ASYNC_RESUME_INFO:
         case GT_LABEL:
         case GT_FTN_ADDR:
-        case GT_RET_EXPR:
         case GT_CNS_INT:
         case GT_CNS_LNG:
         case GT_CNS_DBL:

src/coreclr/jit/fgbasic.cpp

@@ -4728,6 +4728,16 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, Statement* st
     return newBlock;
 }
 
+BasicBlock* Compiler::fgSplitBlockBeforeStatement(BasicBlock* curr, Statement* stmt)
+{
+    if (stmt == curr->firstStmt())
+    {
+        return fgSplitBlockAtBeginning(curr);
+    }
+
+    return fgSplitBlockAfterStatement(curr, stmt->GetPrevStmt());
+}
+
 //------------------------------------------------------------------------------
 // fgSplitBlockBeforeTree : Split the given block right before the given tree
 //