Merge pull request #32131 from slavapestov/binary-swiftdeps-format

New binary swiftdeps format

Author: slavapestov

include/swift/AST/FileSystem.h

@@ -15,6 +15,7 @@
 
 #include "swift/Basic/FileSystem.h"
 #include "swift/AST/DiagnosticEngine.h"
+#include "swift/AST/DiagnosticsCommon.h"
 
 namespace swift {
 

include/swift/AST/FineGrainedDependencies.h

@@ -1,4 +1,4 @@
-//===----- FineGrainedependencies.h -----------------------------*- C++ -*-===//
+//===----- FineGrainedDependencies.h ----------------------------*- C++ -*-===//
 //
 // This source file is part of the Swift.org open source project
 //
@@ -632,6 +632,11 @@ class DepGraphNode {
 
   const DependencyKey &getKey() const { return key; }
 
+  /// Only used when the driver is reading a SourceFileDepGraphNode.
+  void setKey(const DependencyKey &key) {
+    this->key = key;
+  }
+
   const Optional<StringRef> getFingerprint() const {
     if (fingerprint) {
       return StringRef(fingerprint.getValue());
@@ -684,15 +689,15 @@ class SourceFileDepGraphNode : public DepGraphNode {
   /// True iff a Decl exists for this node.
   /// If a provides and a depends in the existing system both have the same key,
   /// only one SourceFileDepGraphNode is emitted.
-  bool isProvides;
+  bool isProvides = false;
 
   friend ::llvm::yaml::MappingContextTraits<SourceFileDepGraphNode,
                                             SourceFileDepGraph>;
 
 public:
   /// When the driver imports a node create an uninitialized instance for
   /// deserializing.
-  SourceFileDepGraphNode() : DepGraphNode(), sequenceNumber(~0) {}
+  SourceFileDepGraphNode() : DepGraphNode() {}
 
   /// Used by the frontend to build nodes.
   SourceFileDepGraphNode(DependencyKey key, Optional<StringRef> fingerprint,
@@ -736,6 +741,9 @@ class SourceFileDepGraphNode : public DepGraphNode {
                                                            : "somewhere else");
   }
 
+  SWIFT_DEBUG_DUMP;
+  void dump(llvm::raw_ostream &os) const;
+
   bool verify() const {
     DepGraphNode::verify();
     assert(getIsProvides() || isDepends());
@@ -872,7 +880,6 @@ class SourceFileDepGraph {
 
   void emitDotFile(StringRef outputPath, DiagnosticEngine &diags);
 
-private:
   void addNode(SourceFileDepGraphNode *n) {
     n->setSequenceNumber(allNodes.size());
     allNodes.push_back(n);

include/swift/AST/FineGrainedDependencyFormat.h

@@ -0,0 +1,133 @@
+//===---- FineGrainedDependencyFormat.h - swiftdeps format ---*- C++ -*-======//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_FINEGRAINEDDEPENDENCYFORMAT_H
+#define SWIFT_AST_FINEGRAINEDDEPENDENCYFORMAT_H
+
+#include "llvm/Bitcode/RecordLayout.h"
+#include "llvm/Bitstream/BitCodes.h"
+
+namespace llvm {
+class MemoryBuffer;
+}
+
+namespace swift {
+
+class DiagnosticEngine;
+
+namespace fine_grained_dependencies {
+
+class SourceFileDepGraph;
+
+using llvm::BCFixed;
+using llvm::BCVBR;
+using llvm::BCBlob;
+using llvm::BCRecordLayout;
+
+/// Every .swiftdeps file begins with these 4 bytes, for easy identification when
+/// debugging.
+const unsigned char FINE_GRAINED_DEPDENENCY_FORMAT_SIGNATURE[] = {'D', 'E', 'P', 'S'};
+
+const unsigned FINE_GRAINED_DEPENDENCY_FORMAT_VERSION_MAJOR = 1;
+
+/// Increment this on every change.
+const unsigned FINE_GRAINED_DEPENDENCY_FORMAT_VERSION_MINOR = 0;
+
+using IdentifierIDField = BCVBR<13>;
+
+using NodeKindField = BCFixed<3>;
+using DeclAspectField = BCFixed<1>;
+
+const unsigned RECORD_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
+
+/// The swiftdeps file format consists of a METADATA record, followed by zero or more
+/// IDENTIFIER_NODE records.
+///
+/// Then, there is one SOURCE_FILE_DEP_GRAPH_NODE for each serialized
+/// SourceFileDepGraphNode. These are followed by FINGERPRINT_NODE and
+/// DEPENDS_ON_DEFINITION_NODE, if the node has a fingerprint and depends-on
+/// definitions, respectively.
+namespace record_block {
+  enum {
+    METADATA = 1,
+    SOURCE_FILE_DEP_GRAPH_NODE,
+    FINGERPRINT_NODE,
+    DEPENDS_ON_DEFINITION_NODE,
+    IDENTIFIER_NODE,
+  };
+
+  // Always the first record in the file.
+  using MetadataLayout = BCRecordLayout<
+    METADATA, // ID
+    BCFixed<16>, // Dependency graph format major version
+    BCFixed<16>, // Dependency graph format minor version
+    BCBlob // Compiler version string
+  >;
+
+  // After the metadata record, we have zero or more identifier records,
+  // for each unique string that is referenced from a SourceFileDepGraphNode.
+  //
+  // Identifiers are referenced by their sequence number, starting from 1.
+  // The identifier value 0 is special; it always represents the empty string.
+  // There is no IDENTIFIER_NODE serialized that corresponds to it, instead
+  // the first IDENTIFIER_NODE always has a sequence number of 1.
+  using IdentifierNodeLayout = BCRecordLayout<
+    IDENTIFIER_NODE,
+    BCBlob
+  >;
+
+  using SourceFileDepGraphNodeLayout = BCRecordLayout<
+    SOURCE_FILE_DEP_GRAPH_NODE, // ID
+    // The next four fields correspond to the fields of the DependencyKey
+    // structure.
+    NodeKindField, // DependencyKey::kind
+    DeclAspectField, // DependencyKey::aspect
+    IdentifierIDField, // DependencyKey::context
+    IdentifierIDField, // DependencyKey::name
+    BCFixed<1> // Is this a "provides" node?
+  >;
+
+  // Follows DEPENDS_ON_DEFINITION_NODE when the SourceFileDepGraphNode has a
+  // fingerprint set.
+  using FingerprintNodeLayout = BCRecordLayout<
+    FINGERPRINT_NODE,
+    BCBlob
+  >;
+
+  // Follows SOURCE_FILE_DEP_GRAPH_NODE and FINGERPRINT_NODE when the
+  // SourceFileDepGraphNode has one or more depends-on entries.
+  using DependsOnDefNodeLayout = BCRecordLayout<
+    DEPENDS_ON_DEFINITION_NODE,
+    BCVBR<16> // The sequence number (starting from 0) of the referenced
+              // SOURCE_FILE_DEP_GRAPH_NODE
+  >;
+}
+
+/// Tries to read the dependency graph from the given buffer.
+/// Returns true if there was an error.
+bool readFineGrainedDependencyGraph(llvm::MemoryBuffer &buffer,
+                                    SourceFileDepGraph &g);
+
+/// Tries to read the dependency graph from the given path name.
+/// Returns true if there was an error.
+bool readFineGrainedDependencyGraph(llvm::StringRef path,
+                                    SourceFileDepGraph &g);
+
+/// Tries to write the dependency graph to the given path name.
+/// Returns true if there was an error.
+bool writeFineGrainedDependencyGraph(DiagnosticEngine &diags, llvm::StringRef path,
+                                     const SourceFileDepGraph &g);
+
+} // namespace fine_grained_dependencies
+} // namespace swift
+
+#endif

lib/AST/CMakeLists.txt

@@ -45,6 +45,7 @@ add_swift_host_library(swiftAST STATIC
   Evaluator.cpp
   Expr.cpp
   FineGrainedDependencies.cpp
+  FineGrainedDependencyFormat.cpp
   FrontendSourceFileDepGraphFactory.cpp
   GenericEnvironment.cpp
   GenericSignature.cpp

lib/AST/FineGrainedDependencies.cpp

@@ -10,15 +10,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <stdio.h>
-
 #include "swift/AST/FineGrainedDependencies.h"
 
 // may not all be needed
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsCommon.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/AST/FileSystem.h"
+#include "swift/AST/FineGrainedDependencyFormat.h"
 #include "swift/Basic/FileSystem.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/Demangling/Demangle.h"
@@ -31,6 +30,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/YAMLParser.h"
 
+
 // This file holds the definitions for the fine-grained dependency system
 // that are likely to be stable as it moves away from the status quo.
 // These include the graph structures common to both programs and also
@@ -53,11 +53,9 @@ Optional<SourceFileDepGraph> SourceFileDepGraph::loadFromPath(StringRef path) {
 Optional<SourceFileDepGraph>
 SourceFileDepGraph::loadFromBuffer(llvm::MemoryBuffer &buffer) {
   SourceFileDepGraph fg;
-  llvm::yaml::Input yamlReader(llvm::MemoryBufferRef(buffer), nullptr);
-  yamlReader >> fg;
-  if (yamlReader.error())
+  if (swift::fine_grained_dependencies::readFineGrainedDependencyGraph(
+      buffer, fg))
     return None;
-  // return fg; compiles for Mac but not Linux, because it cannot be copied.
   return Optional<SourceFileDepGraph>(std::move(fg));
 }
 
@@ -333,6 +331,19 @@ void DepGraphNode::dump(raw_ostream &os) const {
     llvm::errs() << "no fingerprint";
 }
 
+void SourceFileDepGraphNode::dump() const {
+  dump(llvm::errs());
+}
+
+void SourceFileDepGraphNode::dump(raw_ostream &os) const {
+  DepGraphNode::dump(os);
+  os << " sequence number: " << sequenceNumber;
+  os << " is provides: " << isProvides;
+  os << " depends on:";
+  for (auto def : defsIDependUpon)
+    os << " " << def;
+}
+
 std::string DepGraphNode::humanReadableName(StringRef where) const {
 
   return getKey().humanReadableName() +