[SourceKit] Add request to expand macros syntactically

Expand macros in the specified source file syntactically (without any
module imports, nor typechecking).

Request would look like:
```
{
  key.compilerargs: [...]
  key.sourcefile: <file name>
  key.sourcetext: <source text> (optional)
  key.expansions: [<expansion specifier>...]
}
```
`key.compilerargs` are used for getting plugins search paths. If
`key.sourcetext` is not specified, it's loaded from the file system.
Each `<expansion sepecifier>` is
```
{
  key.offset: <offset>
  key.modulename: <plugin module name>
  key.typename: <macro typename>
  key.macro_roles: [<macro role UID>...]
}
```
Clients have to provide the module and type names because that's
semantic.

Response is a `CategorizedEdits` just like (semantic) "ExpandMacro"
refactoring. But without `key.buffer_name`. Nested expnasions are not
supported at this point.

Author: rintaro

include/swift/AST/Decl.h

@@ -8601,6 +8601,9 @@ class MacroDecl : public GenericContext, public ValueDecl {
   /// Retrieve the definition of this macro.
   MacroDefinition getDefinition() const;
 
+  /// Set the definition of this macro
+  void setDefinition(MacroDefinition definition);
+
   /// Retrieve the parameter list of this macro.
   ParameterList *getParameterList() const { return parameterList; }
 

include/swift/IDE/Utils.h

@@ -578,6 +578,9 @@ class SourceEditConsumer {
   void insertAfter(SourceManager &SM, SourceLoc Loc, StringRef Text, ArrayRef<NoteRegion> SubRegions = {});
   void accept(SourceManager &SM, Replacement Replacement) { accept(SM, RegionType::ActiveCode, {Replacement}); }
   void remove(SourceManager &SM, CharSourceRange Range);
+  void acceptMacroExpansionBuffer(SourceManager &SM, unsigned bufferID,
+                                  SourceFile *containingSF,
+                                  bool adjustExpansion, bool includeBufferName);
 };
 
 /// This helper stream inserts text into a SourceLoc by calling functions in

include/swift/IDETool/SyntacticMacroExpansion.h

@@ -0,0 +1,97 @@
+//===--- SyntacticMacroExpansion.h ----------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_IDE_SYNTACTICMACROEXPANSION_H
+#define SWIFT_IDE_SYNTACTICMACROEXPANSION_H
+
+#include "swift/AST/Decl.h"
+#include "swift/AST/MacroDefinition.h"
+#include "swift/AST/PluginRegistry.h"
+#include "swift/Basic/Fingerprint.h"
+#include "swift/Frontend/Frontend.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace swift {
+
+class ASTContext;
+class SourceFile;
+
+namespace ide {
+class SourceEditConsumer;
+
+/// Simple object to specify a syntactic macro expansion.
+struct MacroExpansionSpecifier {
+  unsigned offset;
+  swift::MacroRoles macroRoles;
+  swift::MacroDefinition macroDefinition;
+};
+
+/// Instance of a syntactic macro expansion context. This is created for each
+/// list of compiler arguments (i.e. 'argHash'), and reused as long as the
+/// compiler arguments are not changed.
+class SyntacticMacroExpansionInstance {
+  CompilerInvocation invocation;
+
+  SourceManager SourceMgr;
+  DiagnosticEngine Diags{SourceMgr};
+  std::unique_ptr<ASTContext> Ctx;
+  ModuleDecl *TheModule = nullptr;
+  llvm::StringMap<MacroDecl *> MacroDecls;
+
+  /// Create 'SourceFile' using the buffer.
+  swift::SourceFile *getSourceFile(llvm::MemoryBuffer *inputBuf);
+
+  /// Synthesize 'MacroDecl' AST object to use the expansion.
+  swift::MacroDecl *
+  getSynthesizedMacroDecl(swift::Identifier name,
+                          const MacroExpansionSpecifier &expansion);
+
+  /// Expand single 'expansion' in SF.
+  void expand(swift::SourceFile *SF,
+                    const MacroExpansionSpecifier &expansion,
+                    SourceEditConsumer &consumer);
+
+public:
+  SyntacticMacroExpansionInstance() {}
+
+  /// Setup the instance with \p args .
+  bool setup(StringRef SwiftExecutablePath, ArrayRef<const char *> args,
+             std::shared_ptr<PluginRegistry> plugins, std::string &error);
+
+  ASTContext &getASTContext() { return *Ctx; }
+
+  /// Expand all macros in \p inputBuf and send the edit results to \p consumer.
+  /// Expansions are specified by \p expansions .
+  void expandAll(llvm::MemoryBuffer *inputBuf,
+                     ArrayRef<MacroExpansionSpecifier> expansions,
+                     SourceEditConsumer &consumer);
+};
+
+/// Manager object to vend 'SyntacticMacroExpansionInstance'.
+class SyntacticMacroExpansion {
+  StringRef SwiftExecutablePath;
+  std::shared_ptr<PluginRegistry> Plugins;
+
+public:
+  SyntacticMacroExpansion(StringRef SwiftExecutablePath,
+                          std::shared_ptr<PluginRegistry> Plugins)
+      : SwiftExecutablePath(SwiftExecutablePath), Plugins(Plugins) {}
+
+  /// Get instance configured with the specified compiler arguments.
+  std::shared_ptr<SyntacticMacroExpansionInstance>
+  getInstance(ArrayRef<const char *> args, std::string &error);
+};
+
+} // namespace ide
+} // namespace swift
+
+#endif // SWIFT_IDE_SYNTACTICMACROEXPANSION_H

include/swift/Sema/IDETypeChecking.h

@@ -37,6 +37,7 @@ namespace swift {
   enum class DeclRefKind;
   class Expr;
   class ExtensionDecl;
+  class FreestandingMacroExpansion;
   class FunctionType;
   class LabeledConditionalStmt;
   class LookupResult;
@@ -355,6 +356,13 @@ namespace swift {
   SmallVector<std::pair<ValueDecl *, ValueDecl *>, 1>
   getShorthandShadows(LabeledConditionalStmt *CondStmt,
                       DeclContext *DC = nullptr);
+
+  SourceFile *evaluateFreestandingMacro(FreestandingMacroExpansion *expansion,
+                                        StringRef discriminator);
+
+  SourceFile *evaluateAttachedMacro(MacroDecl *macro, Decl *attachedTo,
+                                    CustomAttr *attr, bool passParentContext,
+                                    MacroRole role, StringRef discriminator);
 }
 
 #endif

lib/AST/Decl.cpp

@@ -10632,6 +10632,11 @@ MacroDefinition MacroDecl::getDefinition() const {
       MacroDefinition::forUndefined());
 }
 
+void MacroDecl::setDefinition(MacroDefinition definition) {
+  getASTContext().evaluator.cacheOutput(MacroDefinitionRequest{this},
+                                        std::move(definition));
+}
+
 Optional<BuiltinMacroKind> MacroDecl::getBuiltinKind() const {
   auto def = getDefinition();
   if (def.kind != MacroDefinition::Kind::Builtin)

lib/IDE/Utils.cpp

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/IDE/Utils.h"
+#include "swift/AST/SourceFile.h"
 #include "swift/Basic/Edit.h"
 #include "swift/Basic/Platform.h"
 #include "swift/Basic/SourceManager.h"
@@ -622,6 +623,102 @@ remove(SourceManager &SM, CharSourceRange Range) {
   accept(SM, Range, "");
 }
 
+/// Given the expanded code for a particular macro, perform whitespace
+/// adjustments to make the refactoring more suitable for inline insertion.
+static StringRef
+adjustMacroExpansionWhitespace(GeneratedSourceInfo::Kind kind,
+                               StringRef expandedCode,
+                               llvm::SmallString<64> &scratch) {
+  scratch.clear();
+
+  switch (kind) {
+  case GeneratedSourceInfo::MemberAttributeMacroExpansion:
+    // Attributes are added to the beginning, add a space to separate from
+    // any existing.
+    scratch += expandedCode;
+    scratch += " ";
+    return scratch;
+
+  case GeneratedSourceInfo::MemberMacroExpansion:
+  case GeneratedSourceInfo::PeerMacroExpansion:
+  case GeneratedSourceInfo::ConformanceMacroExpansion:
+    // All added to the end. Note that conformances are always expanded as
+    // extensions, hence treating them the same as peer.
+    scratch += "\n\n";
+    scratch += expandedCode;
+    scratch += "\n";
+    return scratch;
+
+  case GeneratedSourceInfo::ExpressionMacroExpansion:
+  case GeneratedSourceInfo::FreestandingDeclMacroExpansion:
+  case GeneratedSourceInfo::AccessorMacroExpansion:
+  case GeneratedSourceInfo::ReplacedFunctionBody:
+  case GeneratedSourceInfo::PrettyPrinted:
+    return expandedCode;
+  }
+}
+
+void swift::ide::SourceEditConsumer::acceptMacroExpansionBuffer(
+    SourceManager &SM, unsigned bufferID, SourceFile *containingSF,
+    bool adjustExpansion, bool includeBufferName) {
+  auto generatedInfo = SM.getGeneratedSourceInfo(bufferID);
+  if (!generatedInfo || generatedInfo->originalSourceRange.isInvalid())
+    return;
+
+  auto rewrittenBuffer = SM.extractText(generatedInfo->generatedSourceRange);
+
+  // If there's no change, drop the edit entirely.
+  if (generatedInfo->originalSourceRange.getStart() ==
+          generatedInfo->originalSourceRange.getEnd() &&
+      rewrittenBuffer.empty())
+    return;
+
+  SmallString<64> scratchBuffer;
+  if (adjustExpansion) {
+    rewrittenBuffer = adjustMacroExpansionWhitespace(
+        generatedInfo->kind, rewrittenBuffer, scratchBuffer);
+  }
+
+  // `containingFile` is the file of the actual expansion site, where as
+  // `originalFile` is the possibly enclosing buffer. Concretely:
+  // ```
+  // // m.swift
+  // @AddMemberAttributes
+  // struct Foo {
+  //   // --- expanded from @AddMemberAttributes eg. @_someBufferName ---
+  //   @AddedAttribute
+  //   // ---
+  //   let someMember: Int
+  // }
+  // ```
+  //
+  // When expanding `AddedAttribute`, the expansion actually applies to the
+  // original source (`m.swift`) rather than the buffer of the expansion
+  // site (`@_someBufferName`). Thus, we need to include the path to the
+  // original source as well. Note that this path could itself be another
+  // expansion.
+  auto originalSourceRange = generatedInfo->originalSourceRange;
+  SourceFile *originalFile =
+      containingSF->getParentModule()->getSourceFileContainingLocation(
+          originalSourceRange.getStart());
+  StringRef originalPath;
+  if (originalFile->getBufferID().hasValue() &&
+      containingSF->getBufferID() != originalFile->getBufferID()) {
+    originalPath = SM.getIdentifierForBuffer(*originalFile->getBufferID());
+  }
+
+  StringRef bufferName;
+  if (includeBufferName) {
+    bufferName = SM.getIdentifierForBuffer(bufferID);
+  }
+
+  accept(SM, {originalPath,
+              originalSourceRange,
+              bufferName,
+              rewrittenBuffer,
+              {}});
+}
+
 struct swift::ide::SourceEditJsonConsumer::Implementation {
   llvm::raw_ostream &OS;
   std::vector<SingleEdit> AllEdits;

lib/IDETool/CMakeLists.txt

@@ -4,6 +4,7 @@ add_swift_host_library(swiftIDETool STATIC
   CompilerInvocation.cpp
   IDEInspectionInstance.cpp
   DependencyChecking.cpp
+  SyntacticMacroExpansion.cpp
   )
 
 target_link_libraries(swiftIDETool PRIVATE