Turn swift_asyncLet_finish into a builtin.

Reflect the potential stack allocation of the begin/finish pair in SIL,
which should prevent the optimizer from trying to move stack allocations
around them.

I'm not actually sold that the runtime *has* to do a stack allocation here,
because it only really does it because it creates a result buffer in the
future that it should be able to avoid. But it's undisputable that it
currently does so, and we may come up with good reasons for it to continue
to do so even if we remove the current allocation, so modeling the allocation
is the right thing to do.

Fixes rdar://109850951

Author: rjmccall

include/swift/AST/Builtins.def

@@ -923,6 +923,12 @@ BUILTIN_MISC_OPERATION(FlowSensitiveSelfIsolation, "flowSensitiveSelfIsolation",
 BUILTIN_MISC_OPERATION(FlowSensitiveDistributedSelfIsolation,
                        "flowSensitiveDistributedSelfIsolation", "", Special)
 
+/// finishAsyncLet(Builtin.RawPointer, Builtin.RawPointer) -> Void
+///
+/// End the scope of an async let. An async operation. Deallocates stack
+/// memory that was allocated by startAsyncLetWithLocalBuffer.
+BUILTIN_MISC_OPERATION(FinishAsyncLet, "finishAsyncLet", "", Special)
+
 /// endAsyncLetLifetime(): (Builtin.RawPointer) -> Void
 ///
 /// Marks the end of an async-let's lifetime.

include/swift/AST/Builtins.h

@@ -77,7 +77,7 @@ enum class OverloadedBuiltinKind : uint8_t {
 };
 
 /// BuiltinValueKind - The set of (possibly overloaded) builtin functions.
-enum class BuiltinValueKind {
+enum class BuiltinValueKind : unsigned {
   None = 0,
 #define BUILTIN(Id, Name, Attrs) Id,
 #include "swift/AST/Builtins.def"

include/swift/Runtime/RuntimeFunctions.def

@@ -2534,6 +2534,28 @@ FUNCTION(AsyncLetBegin,
          EFFECT(RuntimeEffect::Concurrency),
          MEMEFFECTS(ArgMemOnly))
 
+/// void swift_asyncLet_finish(
+///     AsyncLet *alet,
+///     TaskOptionRecord *options,
+///     const Metadata *futureResultType,
+///     void *closureEntryPoint,
+///     HeapObject *closureContext,
+///     void *resultBuffer
+/// );
+FUNCTION(AsyncLetFinish,
+         _Concurrency, swift_asyncLet_finish, SwiftAsyncCC,
+         ConcurrencyAvailability,
+         RETURNS(VoidTy),
+         ARGS(SwiftContextPtrTy,          // current context
+              SwiftAsyncLetPtrTy,         // AsyncLet*
+              Int8PtrTy,                  // resultBuffer
+              TaskContinuationFunctionPtrTy, // continuation
+              SwiftContextPtrTy           // temp context
+         ),
+         ATTRS(NoUnwind),
+         EFFECT(RuntimeEffect::Concurrency),
+         MEMEFFECTS(ArgMemOnly))
+
 /// void swift_task_run_inline(
 ///     OpaqueValue *result,
 ///     void *closureAFP,

lib/AST/Builtins.cpp

@@ -1759,6 +1759,12 @@ static ValueDecl *getStartAsyncLet(ASTContext &ctx, Identifier id) {
   return builder.build(id);
 }
 
+static ValueDecl *getFinishAsyncLet(ASTContext &ctx, Identifier id) {
+  return getBuiltinFunction(ctx, id, _async(_thin),
+                            _parameters(_rawPointer, _rawPointer),
+                            _void);
+}
+
 static ValueDecl *getEndAsyncLet(ASTContext &ctx, Identifier id) {
   return getBuiltinFunction(ctx, id, _thin,
                             _parameters(_rawPointer),
@@ -3392,6 +3398,9 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
   case BuiltinValueKind::StartAsyncLetWithLocalBuffer:
     return getStartAsyncLet(Context, Id);
 
+  case BuiltinValueKind::FinishAsyncLet:
+    return getFinishAsyncLet(Context, Id);
+
   case BuiltinValueKind::EndAsyncLetLifetime:
     return getEndAsyncLet(Context, Id);
 

lib/IRGen/Callee.h

@@ -30,6 +30,7 @@ namespace llvm {
 }
 
 namespace swift {
+  enum class BuiltinValueKind : unsigned;
 
 namespace irgen {
   class Callee;
@@ -571,6 +572,13 @@ namespace irgen {
            llvm::Value *firstData = nullptr,
            llvm::Value *secondData = nullptr);
 
+    static Callee forBuiltinRuntimeFunction(IRGenModule &IGM,
+                                            llvm::Constant *fnPtr,
+                                            BuiltinValueKind builtin,
+                                            SubstitutionMap subs,
+                                            FunctionPointerKind fpKind =
+                                              FunctionPointerKind::Function);
+
     SILFunctionTypeRepresentation getRepresentation() const {
       return Info.OrigFnType->getRepresentation();
     }

lib/IRGen/GenBuiltin.cpp

@@ -250,6 +250,13 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin,
     return;
   }
 
+  if (Builtin.ID == BuiltinValueKind::FinishAsyncLet) {
+    auto asyncLet = args.claimNext();
+    auto resultBuffer = args.claimNext();
+    emitFinishAsyncLet(IGF, asyncLet, resultBuffer);
+    return;
+  }
+
   if (Builtin.ID == BuiltinValueKind::EndAsyncLetLifetime) {
     IGF.Builder.CreateLifetimeEnd(args.claimNext());
     // Ignore a second operand which is inserted by ClosureLifetimeFixup and

lib/IRGen/GenCall.cpp

@@ -6596,6 +6596,32 @@ Callee irgen::getCFunctionPointerCallee(IRGenFunction &IGF,
   return Callee(std::move(calleeInfo), fn);
 }
 
+Callee Callee::forBuiltinRuntimeFunction(IRGenModule &IGM,
+                                         llvm::Constant *fnPtr,
+                                         BuiltinValueKind builtin,
+                                         SubstitutionMap subs,
+                                         FunctionPointerKind fpKind) {
+  auto &ctx = IGM.Context;
+  auto builtinDecl =
+    getBuiltinValueDecl(ctx, ctx.getIdentifier(getBuiltinName(builtin)));
+
+  auto loweredFnType = IGM.getSILTypes().getConstantFunctionType(
+                         TypeExpansionContext::minimal(),
+                         SILDeclRef(builtinDecl, SILDeclRef::Kind::Func));
+  auto signature = IGM.getSignature(loweredFnType, fpKind);
+
+  auto fp = FunctionPointer::forDirect(fpKind, fnPtr, nullptr, signature);
+
+  auto substFnType = loweredFnType;
+  if (subs) {
+    substFnType = loweredFnType->substGenericArgs(IGM.getSILModule(), subs,
+                                            TypeExpansionContext::minimal());
+  }
+  CalleeInfo calleeInfo(loweredFnType, substFnType, subs);
+
+  return Callee(std::move(calleeInfo), fp);
+}
+
 FunctionPointer FunctionPointer::forDirect(IRGenModule &IGM,
                                            llvm::Constant *fnPtr,
                                            llvm::Constant *secondaryValue,

lib/IRGen/GenConcurrency.cpp

@@ -18,6 +18,7 @@
 #include "GenConcurrency.h"
 
 #include "BitPatternBuilder.h"
+#include "CallEmission.h"
 #include "ExtraInhabitants.h"
 #include "GenCall.h"
 #include "GenPointerAuth.h"
@@ -312,6 +313,28 @@ llvm::Value *irgen::emitBuiltinStartAsyncLet(IRGenFunction &IGF,
   return alet;
 }
 
+void irgen::emitFinishAsyncLet(IRGenFunction &IGF,
+                               llvm::Value *asyncLet,
+                               llvm::Value *resultBuffer) {
+  llvm::Constant *function = IGF.IGM.getAsyncLetFinishFn();
+  auto callee = Callee::forBuiltinRuntimeFunction(IGF.IGM, function,
+                        BuiltinValueKind::FinishAsyncLet, SubstitutionMap(),
+                        FunctionPointerKind::SpecialKind::AsyncLetFinish);
+  auto emission = getCallEmission(IGF, nullptr, std::move(callee));
+
+  emission->begin();
+
+  Explosion args;
+  args.add(asyncLet);
+  args.add(resultBuffer);
+  emission->setArgs(args, /*outlined*/ false, /*witness metadata*/ nullptr);
+
+  Explosion result;
+  emission->emitToExplosion(result, false);
+
+  emission->end();
+}
+
 llvm::Value *irgen::emitCreateTaskGroup(IRGenFunction &IGF,
                                         SubstitutionMap subs,
                                         llvm::Value *groupFlags) {

lib/IRGen/GenConcurrency.h

@@ -78,6 +78,11 @@ llvm::Value *emitBuiltinStartAsyncLet(IRGenFunction &IGF,
                                       llvm::Value *resultBuffer,
                                       SubstitutionMap subs);
 
+/// Emit the finishAsyncLet builtin.
+void emitFinishAsyncLet(IRGenFunction &IGF,
+                        llvm::Value *asyncLet,
+                        llvm::Value *resultBuffer);
+
 /// Emit the createTaskGroup builtin.
 llvm::Value *emitCreateTaskGroup(IRGenFunction &IGF, SubstitutionMap subs,
                                  llvm::Value *groupFlags);

lib/SIL/IR/OperandOwnership.cpp

@@ -933,6 +933,7 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GetEnumTag)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, InjectEnumTag)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, DistributedActorAsAnyActor)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AddressOfRawLayout)
+BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, FinishAsyncLet)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, EndAsyncLetLifetime)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, CreateTaskGroup)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, CreateTaskGroupWithFlags)