Add table of contents scope management and refactor diagnostics (#774)

* Add table of contents scope management and refactor diagnostics

Introduced `ITableOfContentsScope` to manage scope boundaries, improving link validation and table of contents processing. Updated diagnostics to emit hints for images outside the ToC scope and refactored functions for better reusability. Adjusted related tests and internal structures accordingly.

* Add TODO to make this an error

* post merge build fixes

Author: Mpdreamz

docs/development/index.md

@@ -4,4 +4,7 @@ navigation_title: Development
 
 # Development Guide
 
-TODO write development documentation here
+TODO write development documentation here
+
+
+![Image outside of scope](../images/great-drawing-of-new-structure.png)

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -132,8 +132,7 @@ public static void EmitWarning(this IBlockExtension block, string message)
 		block.Build.Collector.Channel.Write(d);
 	}
 
-
-	public static void EmitError(this InlineProcessor processor, LinkInline inline, string message)
+	private static void LinkDiagnostic(InlineProcessor processor, Severity severity, LinkInline inline, string message)
 	{
 		var url = inline.Url;
 		var line = inline.Line + 1;
@@ -145,36 +144,22 @@ public static void EmitError(this InlineProcessor processor, LinkInline inline,
 			return;
 		var d = new Diagnostic
 		{
-			Severity = Severity.Error,
+			Severity = severity,
 			File = processor.GetContext().MarkdownSourcePath.FullName,
-			Column = column,
+			Column = Math.Max(column, 1),
 			Line = line,
 			Message = message,
 			Length = length
 		};
 		context.Build.Collector.Channel.Write(d);
 	}
 
+	public static void EmitError(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Error, inline, message);
 
-	public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message)
-	{
-		var url = inline.Url;
-		var line = inline.Line + 1;
-		var column = inline.Column;
-		var length = url?.Length ?? 1;
+	public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Warning, inline, message);
 
-		var context = processor.GetContext();
-		if (context.SkipValidation)
-			return;
-		var d = new Diagnostic
-		{
-			Severity = Severity.Warning,
-			File = processor.GetContext().MarkdownSourcePath.FullName,
-			Column = column,
-			Line = line,
-			Message = message,
-			Length = length
-		};
-		context.Build.Collector.Channel.Write(d);
-	}
+	public static void EmitHint(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Hint, inline, message);
 }

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs

@@ -97,11 +97,11 @@ HashSet<string> files
 				if (f.Extension == ".toml")
 				{
 					var rule = DetectionRule.From(f);
-					return new RuleReference(relativePath, detectionRules, true, [], rule);
+					return new RuleReference(Build.Configuration, relativePath, detectionRules, true, [], rule);
 				}
 
 				_ = files.Add(relativePath);
-				return new FileReference(relativePath, true, false, []);
+				return new FileReference(Build.Configuration, relativePath, true, false, []);
 			})
 			.OrderBy(d => d is RuleReference r ? r.Rule.Name : null, StringComparer.OrdinalIgnoreCase)
 			.ToArray();

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs

@@ -6,7 +6,13 @@
 
 namespace Elastic.Markdown.Extensions.DetectionRules;
 
-public record RulesFolderReference(string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+public record RulesFolderReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
 
-public record RuleReference(string Path, string SourceDirectory, bool Found, IReadOnlyCollection<ITocItem> Children, DetectionRule Rule)
-	: FileReference(Path, Found, false, Children);
+public record RuleReference(
+	ITableOfContentsScope TableOfContentsScope,
+	string Path,
+	string SourceDirectory,
+	bool Found,
+	IReadOnlyCollection<ITocItem> Children, DetectionRule Rule
+)
+	: FileReference(TableOfContentsScope, Path, Found, false, Children);

src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.IO.Abstractions;
 using DotNet.Globbing;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Extensions;
@@ -10,7 +11,7 @@
 
 namespace Elastic.Markdown.IO.Configuration;
 
-public record ConfigurationFile : DocumentationFile
+public record ConfigurationFile : DocumentationFile, ITableOfContentsScope
 {
 	private readonly BuildContext _context;
 
@@ -44,6 +45,8 @@ public record ConfigurationFile : DocumentationFile
 	private FeatureFlags? _featureFlags;
 	public FeatureFlags Features => _featureFlags ??= new FeatureFlags(_features);
 
+	public IDirectoryInfo ScopeDirectory { get; }
+
 	/// This is a documentation set that is not linked to by assembler.
 	/// Setting this to true relaxes a few restrictions such as mixing toc references with file and folder reference
 	public bool DevelopmentDocs { get; }
@@ -57,6 +60,7 @@ public ConfigurationFile(BuildContext context)
 		: base(context.ConfigurationPath, context.DocumentationSourceDirectory)
 	{
 		_context = context;
+		ScopeDirectory = context.ConfigurationPath.Directory!;
 		if (!context.ConfigurationPath.Exists)
 		{
 			Project = "unknown";
@@ -122,7 +126,7 @@ public ConfigurationFile(BuildContext context)
 				switch (entry.Key)
 				{
 					case "toc":
-						var toc = new TableOfContentsConfiguration(this, _context, 0, "");
+						var toc = new TableOfContentsConfiguration(this, ScopeDirectory, _context, 0, "");
 						var children = toc.ReadChildren(reader, entry.Entry);
 						var tocEntries = children.OfType<TocReference>().ToArray();
 						if (!DevelopmentDocs && !IsNarrativeDocs && tocEntries.Length > 0 && children.Count != tocEntries.Length)

src/Elastic.Markdown/IO/Configuration/ITocItem.cs

@@ -4,10 +4,16 @@
 
 namespace Elastic.Markdown.IO.Configuration;
 
-public interface ITocItem;
+public interface ITocItem
+{
+	ITableOfContentsScope TableOfContentsScope { get; }
+}
 
-public record FileReference(string Path, bool Found, bool Hidden, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+public record FileReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, bool Hidden, IReadOnlyCollection<ITocItem> Children)
+	: ITocItem;
 
-public record FolderReference(string Path, bool Found, bool InNav, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+public record FolderReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, bool InNav, IReadOnlyCollection<ITocItem> Children)
+	: ITocItem;
 
-public record TocReference(string Path, bool Found, bool InNav, IReadOnlyCollection<ITocItem> Children) : FolderReference(Path, Found, InNav, Children);
+public record TocReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, bool InNav, IReadOnlyCollection<ITocItem> Children)
+	: FolderReference(TableOfContentsScope, Path, Found, InNav, Children);

src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs

@@ -9,22 +9,32 @@
 
 namespace Elastic.Markdown.IO.Configuration;
 
-public record TableOfContentsConfiguration
+public interface ITableOfContentsScope
+{
+	IDirectoryInfo ScopeDirectory { get; }
+}
+
+public record TableOfContentsConfiguration : ITableOfContentsScope
 {
-	private readonly ConfigurationFile _configuration;
 	private readonly BuildContext _context;
+	private readonly int _maxTocDepth;
 	private readonly int _depth;
 	private readonly string _parentPath;
 	private readonly IDirectoryInfo _rootPath;
+	private readonly ConfigurationFile _configuration;
 
 	public HashSet<string> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
 
 	public IReadOnlyCollection<ITocItem> TableOfContents { get; private set; } = [];
 
-	public TableOfContentsConfiguration(ConfigurationFile configuration, BuildContext context, int depth, string parentPath)
+	public IDirectoryInfo ScopeDirectory { get; }
+
+	public TableOfContentsConfiguration(ConfigurationFile configuration, IDirectoryInfo scope, BuildContext context, int depth, string parentPath)
 	{
-		_rootPath = context.DocumentationSourceDirectory;
 		_configuration = configuration;
+		ScopeDirectory = scope;
+		_maxTocDepth = configuration.MaxTocDepth;
+		_rootPath = context.DocumentationSourceDirectory;
 		_context = context;
 		_depth = depth;
 		_parentPath = parentPath;
@@ -33,7 +43,7 @@ public TableOfContentsConfiguration(ConfigurationFile configuration, BuildContex
 	public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string? parentPath = null)
 	{
 		parentPath ??= _parentPath;
-		if (_depth > _configuration.MaxTocDepth)
+		if (_depth > _maxTocDepth)
 		{
 			reader.EmitError($"toc.yml files may only be linked from docset.yml", entry.Key);
 			return [];
@@ -112,7 +122,7 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
 			foreach (var f in toc.Files)
 				_ = Files.Add(f);
 
-			return [new TocReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, toc.TableOfContents)];
+			return [new TocReference(this, $"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, toc.TableOfContents)];
 		}
 
 		if (file is not null)
@@ -133,15 +143,15 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
 					children = extension.CreateTableOfContentItems(parentPath, detectionRules, Files);
 				}
 			}
-			return [new FileReference($"{parentPath}{Path.DirectorySeparatorChar}{file}".TrimStart(Path.DirectorySeparatorChar), fileFound, hiddenFile, children ?? [])];
+			return [new FileReference(this, $"{parentPath}{Path.DirectorySeparatorChar}{file}".TrimStart(Path.DirectorySeparatorChar), fileFound, hiddenFile, children ?? [])];
 		}
 
 		if (folder is not null)
 		{
 			if (children is null)
 				_ = _configuration.ImplicitFolders.Add(parentPath.TrimStart(Path.DirectorySeparatorChar));
 
-			return [new FolderReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, children ?? [])];
+			return [new FolderReference(this, $"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, children ?? [])];
 		}
 
 		return null;
@@ -235,7 +245,7 @@ public IReadOnlyCollection<ITocItem> ReadChildren(YamlStreamReader reader, KeyVa
 			switch (kv.Key)
 			{
 				case "toc":
-					var nestedConfiguration = new TableOfContentsConfiguration(_configuration, _context, _depth + 1, fullTocPath);
+					var nestedConfiguration = new TableOfContentsConfiguration(_configuration, source.Directory!, _context, _depth + 1, fullTocPath);
 					_ = nestedConfiguration.ReadChildren(reader, kv.Entry);
 					return nestedConfiguration;
 			}

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -6,6 +6,7 @@
 using System.Runtime.InteropServices;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Helpers;
+using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Navigation;
 using Elastic.Markdown.Myst;
 using Elastic.Markdown.Myst.Directives;
@@ -45,6 +46,9 @@ DocumentationSet set
 		_configurationFile = build.Configuration.SourceFile;
 		_globalSubstitutions = build.Configuration.Substitutions;
 		_set = set;
+		//may be updated by DocumentationGroup.ProcessTocItems
+		//todo refactor mutability of MarkdownFile as a whole
+		ScopeDirectory = build.Configuration.ScopeDirectory;
 	}
 
 	public string Id { get; } = Guid.NewGuid().ToString("N")[..8];
@@ -80,8 +84,8 @@ public string? NavigationTitle
 	}
 
 	//indexed by slug
-	private readonly Dictionary<string, PageTocItem> _tableOfContent = new(StringComparer.OrdinalIgnoreCase);
-	public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
+	private readonly Dictionary<string, PageTocItem> _pageTableOfContent = new(StringComparer.OrdinalIgnoreCase);
+	public IReadOnlyDictionary<string, PageTocItem> PageTableOfContent => _pageTableOfContent;
 
 	private readonly HashSet<string> _anchors = new(StringComparer.OrdinalIgnoreCase);
 	public IReadOnlySet<string> Anchors => _anchors;
@@ -146,6 +150,8 @@ public string[] YieldParentGroups()
 	/// because we need to minimally parse to see the anchors anchor validation is deferred.
 	public IReadOnlyDictionary<string, string?>? AnchorRemapping { get; set; }
 
+	public IDirectoryInfo ScopeDirectory { get; set; }
+
 	private void ValidateAnchorRemapping()
 	{
 		if (AnchorRemapping is null)
@@ -224,9 +230,9 @@ protected void ReadDocumentInstructions(MarkdownDocument document)
 
 		var toc = GetAnchors(_set, MarkdownParser, YamlFrontMatter, document, subs, out var anchors);
 
-		_tableOfContent.Clear();
+		_pageTableOfContent.Clear();
 		foreach (var t in toc)
-			_tableOfContent[t.Slug] = t;
+			_pageTableOfContent[t.Slug] = t;
 
 
 		foreach (var label in anchors)

src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs

@@ -112,13 +112,15 @@ internal DocumentationGroup(
 					continue;
 
 
-				foreach (var extension in context.Configuration.EnabledExtensions)
-					extension.Visit(d, tocItem);
-
 				md.Parent = this;
 				md.Hidden = file.Hidden;
 				var navigationIndex = Interlocked.Increment(ref fileIndex);
 				md.NavigationIndex = navigationIndex;
+				md.ScopeDirectory = file.TableOfContentsScope.ScopeDirectory;
+
+				foreach (var extension in context.Configuration.EnabledExtensions)
+					extension.Visit(d, tocItem);
+
 
 				if (file.Children.Count > 0 && d is MarkdownFile virtualIndex)
 				{
@@ -153,7 +155,7 @@ internal DocumentationGroup(
 					children =
 					[
 						.. documentationFiles
-							.Select(d => new FileReference(d.RelativePath, true, false, []))
+							.Select(d => new FileReference(folder.TableOfContentsScope, d.RelativePath, true, false, []))
 					];
 				}
 

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -52,7 +52,6 @@ public class DiagnosticLinkInlineParser : LinkInlineParser
 	public override bool Match(InlineProcessor processor, ref StringSlice slice)
 	{
 		var match = base.Match(processor, ref slice);
-
 		if (!match || processor.Inline is not LinkInline link)
 			return match;
 
@@ -72,6 +71,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
 
 	private static void ParseStylingInstructions(LinkInline link)
 	{
+		if (!link.IsImage)
+			return;
+
 		if (string.IsNullOrWhiteSpace(link.Title) || link.Title.IndexOf('=') < 0)
 			return;
 
@@ -189,7 +191,16 @@ private static void ProcessInternalLink(LinkInline link, InlineProcessor process
 		var includeFrom = GetIncludeFromPath(url, context);
 		var file = ResolveFile(context, url);
 		ValidateInternalUrl(processor, url, includeFrom, link, context);
-		ProcessLinkText(processor, link, context, url, anchor, file);
+
+		if (link.IsImage && context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown)
+		{
+			//TODO make this an error once all offending repositories have been updated
+			if (!file.Directory!.FullName.StartsWith(currentMarkdown.ScopeDirectory.FullName + Path.DirectorySeparatorChar))
+				processor.EmitHint(link, $"Image '{url}' is referenced out of table of contents scope '{currentMarkdown.ScopeDirectory}'.");
+		}
+
+		var linkMarkdown = context.DocumentationFileLookup(file) as MarkdownFile;
+		ProcessLinkText(processor, link, linkMarkdown, anchor, url, file);
 		UpdateLinkUrl(link, url, context, anchor, file);
 	}
 
@@ -214,14 +225,12 @@ private static void ValidateInternalUrl(InlineProcessor processor, string url, s
 			processor.EmitError(link, $"`{url}` does not exist. resolved to `{pathOnDisk}");
 	}
 
-	private static void ProcessLinkText(InlineProcessor processor, LinkInline link, ParserContext context, string url, string? anchor, IFileInfo file)
+	private static void ProcessLinkText(InlineProcessor processor, LinkInline link, MarkdownFile? markdown, string? anchor, string url, IFileInfo file)
 	{
 		if (link.FirstChild != null && string.IsNullOrEmpty(anchor))
 			return;
 
-		var markdown = context.DocumentationFileLookup(file) as MarkdownFile;
-
-		if (markdown == null && link.FirstChild == null)
+		if (markdown is null && link.FirstChild == null)
 		{
 			processor.EmitWarning(link,
 				$"'{url}' could not be resolved to a markdown file while creating an auto text link, '{file.FullName}' does not exist.");
@@ -234,7 +243,7 @@ private static void ProcessLinkText(InlineProcessor processor, LinkInline link,
 		{
 			if (markdown is not null)
 				ValidateAnchor(processor, markdown, anchor, link);
-			if (link.FirstChild == null && (markdown?.TableOfContents.TryGetValue(anchor, out var heading) ?? false))
+			if (link.FirstChild == null && (markdown?.PageTableOfContent.TryGetValue(anchor, out var heading) ?? false))
 				title += " > " + heading.Heading;
 		}
 

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -92,7 +92,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, MarkdownDocument d
 			Title = markdown.Title ?? "[TITLE NOT SET]",
 			TitleRaw = markdown.TitleRaw ?? "[TITLE NOT SET]",
 			MarkdownHtml = html,
-			PageTocItems = [.. markdown.TableOfContents.Values],
+			PageTocItems = [.. markdown.PageTableOfContent.Values],
 			Tree = DocumentationSet.Tree,
 			CurrentDocument = markdown,
 			PreviousDocument = previous,