[SourceKit] Add request to expand macros syntactically

Expand macros in the specified source file syntactically (without any
module imports, nor typechecking).

Request would look like:
```
{
  key.compilerargs: [...]
  key.sourcefile: <file name>
  key.sourcetext: <source text> (optional)
  key.expansions: [<expansion specifier>...]
}
```
`key.compilerargs` are used for getting plugins search paths. If
`key.sourcetext` is not specified, it's loaded from the file system.
Each `<expansion sepecifier>` is
```
{
  key.offset: <offset>
  key.modulename: <plugin module name>
  key.typename: <macro typename>
  key.macro_roles: [<macro role UID>...]
}
```
Clients have to provide the module and type names because that's
semantic.

Response is a `CategorizedEdits` just like (semantic) "ExpandMacro"
refactoring. Each edit object has `key.buffer_name` that can be used for
recursive expansion. If the client founds nested macro expansion in the
expanded source, it can send another request with the same compiler
arguments using the buffer name and the source to get the nested
expansions.

Author: rintaro

include/swift/AST/Decl.h

@@ -8563,6 +8563,9 @@ class MacroDecl : public GenericContext, public ValueDecl {
   /// Retrieve the definition of this macro.
   MacroDefinition getDefinition() const;
 
+  /// Set the definition of this macro
+  void setDefinition(MacroDefinition definition);
+
   /// Retrieve the parameter list of this macro.
   ParameterList *getParameterList() const { return parameterList; }
 

include/swift/IDE/Utils.h

@@ -578,6 +578,9 @@ class SourceEditConsumer {
   void insertAfter(SourceManager &SM, SourceLoc Loc, StringRef Text, ArrayRef<NoteRegion> SubRegions = {});
   void accept(SourceManager &SM, Replacement Replacement) { accept(SM, RegionType::ActiveCode, {Replacement}); }
   void remove(SourceManager &SM, CharSourceRange Range);
+  void acceptMacroExpansionBuffer(SourceManager &SM, unsigned bufferID,
+                                  SourceFile *containingSF,
+                                  bool adjustExpansion);
 };
 
 /// This helper stream inserts text into a SourceLoc by calling functions in

include/swift/IDETool/SyntacticMacroExpansion.h

@@ -0,0 +1,105 @@
+//===--- SyntacticMacroExpansion.h ----------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_IDE_SYNTACTICMACROEXPANSION_H
+#define SWIFT_IDE_SYNTACTICMACROEXPANSION_H
+
+#include "swift/AST/Decl.h"
+#include "swift/AST/MacroDefinition.h"
+#include "swift/AST/PluginRegistry.h"
+#include "swift/Basic/Fingerprint.h"
+#include "swift/Frontend/Frontend.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace swift {
+
+class ASTContext;
+class SourceFile;
+
+namespace ide {
+class SourceEditConsumer;
+
+/// Simple object to specify a syntactic macro expansion.
+struct MacroExpansionSpecifier {
+  unsigned offset;
+  swift::MacroRoles macroRoles;
+  swift::MacroDefinition macroDefinition;
+};
+
+/// Instance of a syntactic macro expansion context. This is created for each
+/// list of compiler arguments (i.e. 'argHash'), and reused as long as the
+/// compiler arguments are not changed.
+class SyntacticMacroExpansionInstance {
+  const Fingerprint argHash;
+  CompilerInvocation invocation;
+
+  SourceManager SourceMgr;
+  DiagnosticEngine Diags{SourceMgr};
+  std::unique_ptr<ASTContext> Ctx;
+  ModuleDecl *TheModule = nullptr;
+  llvm::DenseMap<Identifier, MacroDecl *> MacroDecls;
+
+  std::mutex mtx;
+
+  /// Create 'SourceFile' using the buffer.
+  swift::SourceFile *getSourceFile(llvm::MemoryBuffer *inputBuf);
+
+  /// Synthesize 'MacroDecl' AST object to use the expansion.
+  swift::MacroDecl *
+  getSynthesizedMacroDecl(swift::Identifier name,
+                          const MacroExpansionSpecifier &expansion);
+
+  /// Expand single 'expansion' in SF.
+  bool getExpansion(swift::SourceFile *SF,
+                    const MacroExpansionSpecifier &expansion,
+                    SourceEditConsumer &consumer);
+
+public:
+  SyntacticMacroExpansionInstance(Fingerprint argHash) : argHash(argHash) {}
+
+  /// Setup the instance with \p args .
+  bool setup(StringRef SwiftExecutablePath, ArrayRef<const char *> args,
+             std::shared_ptr<PluginRegistry> plugins, std::string &error);
+
+  const Fingerprint &getArgHash() const { return argHash; }
+  ASTContext &getASTContext() { return *Ctx; }
+
+  /// Expand all macros in \p inputBuf and send the edit results to \p consumer.
+  /// Expansions are specified by \p expansions .
+  bool getExpansions(llvm::MemoryBuffer *inputBuf,
+                     ArrayRef<MacroExpansionSpecifier> expansions,
+                     SourceEditConsumer &consumer);
+};
+
+/// Manager object to vend 'SyntacticMacroExpansionInstance'.
+class SyntacticMacroExpansion {
+  StringRef SwiftExecutablePath;
+  std::shared_ptr<PluginRegistry> Plugins;
+
+  /// Cached instance.
+  std::shared_ptr<SyntacticMacroExpansionInstance> currentInstance;
+
+public:
+  SyntacticMacroExpansion(StringRef SwiftExecutablePath,
+                          std::shared_ptr<PluginRegistry> Plugins)
+      : SwiftExecutablePath(SwiftExecutablePath), Plugins(Plugins) {}
+
+  /// Get instance configured with the specified compiler arguments.
+  /// If 'currentInstance' matches with the arguments, just return it.
+  std::shared_ptr<SyntacticMacroExpansionInstance>
+  getInstance(ArrayRef<const char *> args, std::string &error);
+};
+
+} // namespace ide
+} // namespace swift
+
+#endif // SWIFT_IDE_SYNTACTICMACROEXPANSION_H

include/swift/Sema/IDETypeChecking.h

@@ -37,6 +37,7 @@ namespace swift {
   enum class DeclRefKind;
   class Expr;
   class ExtensionDecl;
+  class FreestandingMacroExpansion;
   class FunctionType;
   class LabeledConditionalStmt;
   class LookupResult;
@@ -355,6 +356,13 @@ namespace swift {
   SmallVector<std::pair<ValueDecl *, ValueDecl *>, 1>
   getShorthandShadows(LabeledConditionalStmt *CondStmt,
                       DeclContext *DC = nullptr);
+
+  SourceFile *evaluateFreestandingMacro(FreestandingMacroExpansion *expansion,
+                                        StringRef discriminator);
+
+  SourceFile *evaluateAttachedMacro(MacroDecl *macro, Decl *attachedTo,
+                                    CustomAttr *attr, bool passParentContext,
+                                    MacroRole role, StringRef discriminator);
 }
 
 #endif

lib/AST/Decl.cpp

@@ -10581,6 +10581,11 @@ MacroDefinition MacroDecl::getDefinition() const {
       MacroDefinition::forUndefined());
 }
 
+void MacroDecl::setDefinition(MacroDefinition definition) {
+  getASTContext().evaluator.cacheOutput(MacroDefinitionRequest{this},
+                                        std::move(definition));
+}
+
 Optional<BuiltinMacroKind> MacroDecl::getBuiltinKind() const {
   auto def = getDefinition();
   if (def.kind != MacroDefinition::Kind::Builtin)

lib/IDE/Utils.cpp

@@ -622,6 +622,97 @@ remove(SourceManager &SM, CharSourceRange Range) {
   accept(SM, Range, "");
 }
 
+/// Given the expanded code for a particular macro, perform whitespace
+/// adjustments to make the refactoring more suitable for inline insertion.
+static StringRef
+adjustMacroExpansionWhitespace(GeneratedSourceInfo::Kind kind,
+                               StringRef expandedCode,
+                               llvm::SmallString<64> &scratch) {
+  scratch.clear();
+
+  switch (kind) {
+  case GeneratedSourceInfo::MemberAttributeMacroExpansion:
+    // Attributes are added to the beginning, add a space to separate from
+    // any existing.
+    scratch += expandedCode;
+    scratch += " ";
+    return scratch;
+
+  case GeneratedSourceInfo::MemberMacroExpansion:
+  case GeneratedSourceInfo::PeerMacroExpansion:
+  case GeneratedSourceInfo::ConformanceMacroExpansion:
+    // All added to the end. Note that conformances are always expanded as
+    // extensions, hence treating them the same as peer.
+    scratch += "\n\n";
+    scratch += expandedCode;
+    scratch += "\n";
+    return scratch;
+
+  case GeneratedSourceInfo::ExpressionMacroExpansion:
+  case GeneratedSourceInfo::FreestandingDeclMacroExpansion:
+  case GeneratedSourceInfo::AccessorMacroExpansion:
+  case GeneratedSourceInfo::ReplacedFunctionBody:
+  case GeneratedSourceInfo::PrettyPrinted:
+    return expandedCode;
+  }
+}
+
+void swift::ide::SourceEditConsumer::acceptMacroExpansionBuffer(
+    SourceManager &SM, unsigned bufferID, SourceFile *containingSF,
+    bool adjustExpansion) {
+  auto generatedInfo = SM.getGeneratedSourceInfo(bufferID);
+  if (!generatedInfo || generatedInfo->originalSourceRange.isInvalid())
+    return;
+
+  auto rewrittenBuffer = SM.extractText(generatedInfo->generatedSourceRange);
+
+  // If there's no change, drop the edit entirely.
+  if (generatedInfo->originalSourceRange.getStart() ==
+          generatedInfo->originalSourceRange.getEnd() &&
+      rewrittenBuffer.empty())
+    return;
+
+  SmallString<64> scratchBuffer;
+  if (adjustExpansion) {
+    rewrittenBuffer = adjustMacroExpansionWhitespace(
+        generatedInfo->kind, rewrittenBuffer, scratchBuffer);
+  }
+
+  // `containingFile` is the file of the actual expansion site, where as
+  // `originalFile` is the possibly enclosing buffer. Concretely:
+  // ```
+  // // m.swift
+  // @AddMemberAttributes
+  // struct Foo {
+  //   // --- expanded from @AddMemberAttributes eg. @_someBufferName ---
+  //   @AddedAttribute
+  //   // ---
+  //   let someMember: Int
+  // }
+  // ```
+  //
+  // When expanding `AddedAttribute`, the expansion actually applies to the
+  // original source (`m.swift`) rather than the buffer of the expansion
+  // site (`@_someBufferName`). Thus, we need to include the path to the
+  // original source as well. Note that this path could itself be another
+  // expansion.
+  auto originalSourceRange = generatedInfo->originalSourceRange;
+  SourceFile *originalFile =
+      containingSF->getParentModule()->getSourceFileContainingLocation(
+          originalSourceRange.getStart());
+  StringRef originalPath;
+  if (originalFile->getBufferID().hasValue() &&
+      containingSF->getBufferID() != originalFile->getBufferID()) {
+    originalPath = SM.getIdentifierForBuffer(*originalFile->getBufferID());
+  }
+
+  accept(SM, {originalPath,
+              originalSourceRange,
+              SM.getIdentifierForBuffer(bufferID),
+              rewrittenBuffer,
+              {}});
+}
+
 struct swift::ide::SourceEditJsonConsumer::Implementation {
   llvm::raw_ostream &OS;
   std::vector<SingleEdit> AllEdits;

lib/IDETool/CMakeLists.txt

@@ -4,6 +4,7 @@ add_swift_host_library(swiftIDETool STATIC
   CompilerInvocation.cpp
   IDEInspectionInstance.cpp
   DependencyChecking.cpp
+  SyntacticMacroExpansion.cpp
   )
 
 target_link_libraries(swiftIDETool PRIVATE