Support resolving documentation links to content in other DocC archives

Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift

@@ -46,11 +46,15 @@ public protocol ConvertOutputConsumer {
 
     /// Consumes build metadata created during a conversion.
     func consume(buildMetadata: BuildMetadata) throws
+
+    /// Consumes a file representation of the local link resolution information.
+    func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws
 }
 
 // Default implementations that discard the documentation conversion products, for consumers that don't need these
 // values.
 public extension ConvertOutputConsumer {
     func consume(renderReferenceStore: RenderReferenceStore) throws {}
     func consume(buildMetadata: BuildMetadata) throws {}
+    func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws {}
 }

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -110,11 +110,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         }
     }
 
-    /// A link resolver that resolves references by finding them in path hierarchy.
-    ///
-    /// The link resolver is `nil` until some documentation content is registered with the context.
-    /// It's safe to access the link resolver during symbol registration and at later points in the registration and conversion.
-    var hierarchyBasedLinkResolver: PathHierarchyBasedLinkResolver! = nil
+    /// A class that resolves documentation links by orchestrating calls to other link resolver implementations.
+    public var linkResolver = LinkResolver()
 
     /// The provider of documentation bundles for this context.
     var dataProvider: DocumentationContextDataProvider
@@ -200,6 +197,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     /// A list of non-topic links that can be resolved.
     var nodeAnchorSections = [ResolvedTopicReference: AnchorSection]()
 
+    var externalCache = [ResolvedTopicReference: LinkResolver.ExternalEntity]()
+
     /// A list of all the problems that was encountered while registering and processing the documentation bundles in this context.
     public var problems: [Problem] {
         return diagnosticEngine.problems
@@ -361,7 +360,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     ///   - dataProvider: The provider that removed this bundle.
     ///   - bundle: The bundle that was removed.
     public func dataProvider(_ dataProvider: DocumentationContextDataProvider, didRemoveBundle bundle: DocumentationBundle) throws {
-        hierarchyBasedLinkResolver?.unregisterBundle(identifier: bundle.identifier)
+        linkResolver.localResolver?.unregisterBundle(identifier: bundle.identifier)
 
         // Purge the reference cache for this bundle and disable reference caching for
         // this bundle moving forward.
@@ -1153,7 +1152,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             var moduleReferences = [String: ResolvedTopicReference]()
 
             // Build references for all symbols in all of this module's symbol graphs.
-            let symbolReferences = hierarchyBasedLinkResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
+            let symbolReferences = linkResolver.localResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
 
             // Set the index and cache storage capacity to avoid ad-hoc storage resizing.
             symbolIndex.reserveCapacity(symbolReferences.count)
@@ -1270,7 +1269,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             // Only add the symbol mapping now if the path hierarchy based resolver is the main implementation.
             // If it is only used for mismatch checking then we must wait until the documentation cache code path has traversed and updated all the colliding nodes.
             // Otherwise the mappings will save the unmodified references and the hierarchy based resolver won't find the expected parent nodes when resolving links.
-            hierarchyBasedLinkResolver.addMappingForSymbols(symbolIndex: symbolIndex)
+            linkResolver.localResolver.addMappingForSymbols(symbolIndex: symbolIndex)
 
             // Track the symbols that have multiple matching documentation extension files for diagnostics.
             var symbolsWithMultipleDocumentationExtensionMatches = [ResolvedTopicReference: [SemanticResult<Article>]]()
@@ -1817,7 +1816,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             topicGraph.addNode(graphNode)
             documentationCache[reference] = documentation
 
-            hierarchyBasedLinkResolver.addRootArticle(article, anchorSections: documentation.anchorSections)
+            linkResolver.localResolver.addRootArticle(article, anchorSections: documentation.anchorSections)
             for anchor in documentation.anchorSections {
                 nodeAnchorSections[anchor.reference] = anchor
             }
@@ -1873,7 +1872,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             let graphNode = TopicGraph.Node(reference: reference, kind: .article, source: .file(url: article.source), title: title)
             topicGraph.addNode(graphNode)
 
-            hierarchyBasedLinkResolver.addArticle(article, anchorSections: documentation.anchorSections)
+            linkResolver.localResolver.addArticle(article, anchorSections: documentation.anchorSections)
             for anchor in documentation.anchorSections {
                 nodeAnchorSections[anchor.reference] = anchor
             }
@@ -2078,6 +2077,17 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             }
         }
 
+        discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
+            do {
+                try linkResolver.loadExternalResolvers()
+            } catch {
+                // Pipe the error out of the dispatch queue.
+                discoveryError.sync({
+                    if $0 == nil { $0 = error }
+                })
+            }
+        }
+
         discoveryGroup.wait()
 
         try shouldContinueRegistration()
@@ -2128,7 +2138,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             options = globalOptions.first
         }
 
-        self.hierarchyBasedLinkResolver = hierarchyBasedResolver
+        self.linkResolver.localResolver = hierarchyBasedResolver
         hierarchyBasedResolver.addMappingForRoots(bundle: bundle)
         for tutorial in tutorials {
             hierarchyBasedResolver.addTutorial(tutorial)
@@ -2187,7 +2197,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         }
 
         try shouldContinueRegistration()
-        var allCuratedReferences = try crawlSymbolCuration(in: hierarchyBasedLinkResolver.topLevelSymbols(), bundle: bundle)
+        var allCuratedReferences = try crawlSymbolCuration(in: linkResolver.localResolver.topLevelSymbols(), bundle: bundle)
 
         // Store the list of manually curated references if doc coverage is on.
         if shouldStoreManuallyCuratedReferences {
@@ -2222,7 +2232,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         // Emit warnings for any remaining uncurated files.
         emitWarningsForUncuratedTopics()
 
-        hierarchyBasedLinkResolver.addAnchorForSymbols(symbolIndex: symbolIndex, documentationCache: documentationCache)
+        linkResolver.localResolver.addAnchorForSymbols(symbolIndex: symbolIndex, documentationCache: documentationCache)
 
         // Fifth, resolve links in nodes that are added solely via curation
         try preResolveExternalLinks(references: Array(allCuratedReferences), bundle: bundle)
@@ -2329,7 +2339,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     /// - Returns: An ordered list of symbol references that have been added to the topic graph automatically.
     private func autoCurateSymbolsInTopicGraph(engine: DiagnosticEngine) -> [(child: ResolvedTopicReference, parent: ResolvedTopicReference)] {
         var automaticallyCuratedSymbols = [(ResolvedTopicReference, ResolvedTopicReference)]()
-        hierarchyBasedLinkResolver.traverseSymbolAndParentPairs { reference, parentReference in
+        linkResolver.localResolver.traverseSymbolAndParentPairs { reference, parentReference in
             guard let topicGraphNode = topicGraph.nodeWithReference(reference),
                   let topicGraphParentNode = topicGraph.nodeWithReference(parentReference),
                   // Check that the node hasn't got any parents from manual curation
@@ -2535,6 +2545,9 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             let referenceWithoutFragment = reference.withFragment(nil)
             return try entity(with: referenceWithoutFragment).availableSourceLanguages
         } catch ContextError.notFound {
+            if let externalEntity = externalCache[reference] {
+                return externalEntity.sourceLanguages
+            }
             preconditionFailure("Reference does not have an associated documentation node.")
         } catch {
             fatalError("Unexpected error when retrieving source languages: \(error)")
@@ -2646,7 +2659,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     public func resolve(_ reference: TopicReference, in parent: ResolvedTopicReference, fromSymbolLink isCurrentlyResolvingSymbolLink: Bool = false) -> TopicReferenceResolutionResult {
         switch reference {
         case .unresolved(let unresolvedReference):
-            return hierarchyBasedLinkResolver.resolve(unresolvedReference, in: parent, fromSymbolLink: isCurrentlyResolvingSymbolLink, context: self)
+            return linkResolver.resolve(unresolvedReference, in: parent, fromSymbolLink: isCurrentlyResolvingSymbolLink, context: self)
+
         case .resolved(let resolved):
             // This reference is already resolved (either as a success or a failure), so don't change anything.
             return resolved

Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift

@@ -330,14 +330,20 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
                     }
 
                     if emitDigest {
-                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode)
+                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true)
                         let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier)
 
                         resultsGroup.async(queue: resultsSyncQueue) {
                             assets.merge(renderNode.assetReferences, uniquingKeysWith: +)
                             linkSummaries.append(contentsOf: nodeLinkSummaries)
                             indexingRecords.append(contentsOf: nodeIndexingRecords)
                         }
+                    } else if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
+                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false)
+
+                        resultsGroup.async(queue: resultsSyncQueue) {
+                            linkSummaries.append(contentsOf: nodeLinkSummaries)
+                        }
                     }
                 } catch {
                     recordProblem(from: error, in: &results, withIdentifier: "render-node")
@@ -362,6 +368,19 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
             }
         }
 
+        if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
+            do {
+                let serializableLinkInformation = try context.linkResolver.localResolver.prepareForSerialization(bundleID: bundle.identifier)
+                try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation)
+
+                if !emitDigest {
+                    try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                }
+            } catch {
+                recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver")
+            }
+        }
+
         if emitDigest {
             do {
                 try outputConsumer.consume(problems: context.problems + conversionProblems)

Sources/SwiftDocC/Infrastructure/DocumentationCurator.swift

@@ -115,7 +115,7 @@ struct DocumentationCurator {
         context.topicGraph.addNode(curatedNode)
 
         // Move the article from the article cache to the documentation
-        context.hierarchyBasedLinkResolver.addArticle(filename: articleFilename, reference: reference, anchorSections: documentationNode.anchorSections)
+        context.linkResolver.localResolver.addArticle(filename: articleFilename, reference: reference, anchorSections: documentationNode.anchorSections)
 
         context.documentationCache[reference] = documentationNode
         for anchor in documentationNode.anchorSections {