From e71e3ba864d7b9235ba3d6c2f816953930c2fef6 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 14:58:45 +0200 Subject: [PATCH 01/11] Add caching --- .../DocumentationGenerator.cs | 11 ++ .../Myst/Directives/Diagram/DiagramBlock.cs | 81 ++++++++++ .../Directives/Diagram/DiagramRegistry.cs | 146 ++++++++++++++++++ .../Directives/Diagram/DiagramView.cshtml | 12 +- .../Directives/DiagramTests.cs | 96 +++++++++++- 5 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 253dee397..3a89faefc 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -16,6 +16,7 @@ using Elastic.Markdown.Helpers; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; +using Elastic.Markdown.Myst.Directives.Diagram; using Elastic.Markdown.Myst.Renderers; using Elastic.Markdown.Myst.Renderers.LlmMarkdown; using Markdig.Syntax; @@ -106,6 +107,9 @@ public async Task ResolveDirectoryTree(Cancel ctx) public async Task GenerateAll(Cancel ctx) { + // Clear diagram registry for fresh tracking + DiagramRegistry.Clear(); + var result = new GenerationResult(); var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState(); @@ -142,6 +146,13 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating links.json"); var linkReference = await GenerateLinkReference(ctx); + // Clean up unused diagram files + var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory.FullName); + if (cleanedCount > 0) + { + _logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount); + } + // ReSharper disable once WithExpressionModifiesAllMembers return result with { diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index f9a7c75fb..a85c250da 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -2,6 +2,8 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Security.Cryptography; +using System.Text; using Elastic.Markdown.Diagnostics; namespace Elastic.Markdown.Myst.Directives.Diagram; @@ -25,6 +27,16 @@ public class DiagramBlock(DirectiveBlockParser parser, ParserContext context) : /// public string? EncodedUrl { get; private set; } + /// + /// The local SVG path relative to the output directory + /// + public string? LocalSvgPath { get; private set; } + + /// + /// Content hash for unique identification and caching + /// + public string? ContentHash { get; private set; } + public override void FinalizeAndValidate(ParserContext context) { // Extract diagram type from arguments or default to "mermaid" @@ -39,6 +51,12 @@ public override void FinalizeAndValidate(ParserContext context) return; } + // Generate content hash for caching + ContentHash = GenerateContentHash(DiagramType, Content); + + // Generate local path for cached SVG + LocalSvgPath = GenerateLocalPath(context); + // Generate the encoded URL for Kroki try { @@ -47,7 +65,14 @@ public override void FinalizeAndValidate(ParserContext context) catch (Exception ex) { this.EmitError($"Failed to encode diagram: {ex.Message}", ex); + return; } + + // Register diagram for tracking and cleanup + DiagramRegistry.RegisterDiagram(LocalSvgPath); + + // Cache diagram asynchronously + _ = Task.Run(() => TryCacheDiagramAsync(context)); } private string? ExtractContent() @@ -68,4 +93,60 @@ public override void FinalizeAndValidate(ParserContext context) return lines.Count > 0 ? string.Join("\n", lines) : null; } + + private string GenerateContentHash(string diagramType, string content) + { + var input = $"{diagramType}:{content}"; + var bytes = Encoding.UTF8.GetBytes(input); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash)[..12].ToLowerInvariant(); + } + + private string GenerateLocalPath(ParserContext context) + { + var markdownFileName = "unknown"; + if (context.MarkdownSourcePath?.FullName != null) + { + markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.FullName); + } + + var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg"; + return Path.Combine("images", "generated-graphs", filename); + } + + private async Task TryCacheDiagramAsync(ParserContext context) + { + if (string.IsNullOrEmpty(EncodedUrl) || string.IsNullOrEmpty(LocalSvgPath)) + return; + + try + { + // Determine the full output path + var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; + var fullPath = Path.Combine(outputDirectory, LocalSvgPath); + + // Skip if file already exists + if (File.Exists(fullPath)) + return; + + // Create directory if it doesn't exist + var directory = Path.GetDirectoryName(fullPath); + if (directory != null && !Directory.Exists(directory)) + { + _ = Directory.CreateDirectory(directory); + } + + // Download SVG from Kroki + using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(30) }; + var svgContent = await httpClient.GetStringAsync(EncodedUrl); + + // Write to local file + await File.WriteAllTextAsync(fullPath, svgContent); + } + catch + { + // Silent failure - caching is opportunistic + // The system will fall back to Kroki URLs + } + } } diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs new file mode 100644 index 000000000..351c6a9bd --- /dev/null +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; + +namespace Elastic.Markdown.Myst.Directives.Diagram; + +/// +/// Registry to track active diagrams and manage cleanup of outdated cached files +/// +public static class DiagramRegistry +{ + private static readonly HashSet ActiveDiagrams = []; + private static readonly Lock Lock = new(); + + /// + /// Register a diagram as active during the current build + /// + /// The local SVG path relative to output directory + public static void RegisterDiagram(string localSvgPath) + { + if (string.IsNullOrEmpty(localSvgPath)) + return; + + lock (Lock) + { + _ = ActiveDiagrams.Add(localSvgPath); + } + } + + /// + /// Get all currently registered active diagrams + /// + /// Collection of active diagram paths + public static IReadOnlyCollection GetActiveDiagrams() + { + lock (Lock) + { + return ActiveDiagrams.ToArray(); + } + } + + /// + /// Clear all registered diagrams (typically called at start of build) + /// + public static void Clear() + { + lock (Lock) + { + ActiveDiagrams.Clear(); + } + } + + /// + /// Clean up unused diagram files from the output directory + /// + /// The output directory path + /// Number of files cleaned up + public static int CleanupUnusedDiagrams(string outputDirectory) => + CleanupUnusedDiagrams(outputDirectory, new FileSystem()); + + /// + /// Clean up unused diagram files from the output directory + /// + /// The output directory path + /// File system abstraction for testing + /// Number of files cleaned up + public static int CleanupUnusedDiagrams(string outputDirectory, IFileSystem fileSystem) + { + if (string.IsNullOrEmpty(outputDirectory)) + return 0; + + var graphsDir = fileSystem.Path.Combine(outputDirectory, "images", "generated-graphs"); + if (!fileSystem.Directory.Exists(graphsDir)) + return 0; + + var cleanedCount = 0; + var activePaths = GetActiveDiagrams(); + + try + { + var existingFiles = fileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories); + + foreach (var file in existingFiles) + { + var relativePath = fileSystem.Path.GetRelativePath(outputDirectory, file); + + // Convert to forward slashes for consistent comparison + var normalizedPath = relativePath.Replace(fileSystem.Path.DirectorySeparatorChar, '/'); + + if (!activePaths.Any(active => active.Replace(fileSystem.Path.DirectorySeparatorChar, '/') == normalizedPath)) + { + try + { + fileSystem.File.Delete(file); + cleanedCount++; + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + } + + // Clean up empty directories + CleanupEmptyDirectories(graphsDir, fileSystem); + } + catch + { + // Silent failure - cleanup is opportunistic + } + + return cleanedCount; + } + + /// + /// Remove empty directories recursively + /// + /// Directory to clean up + /// File system abstraction + private static void CleanupEmptyDirectories(string directory, IFileSystem fileSystem) + { + try + { + if (!fileSystem.Directory.Exists(directory)) + return; + + // Clean up subdirectories first + foreach (var subDir in fileSystem.Directory.GetDirectories(directory)) + { + CleanupEmptyDirectories(subDir, fileSystem); + } + + // Remove directory if it's empty + if (!fileSystem.Directory.EnumerateFileSystemEntries(directory).Any()) + { + fileSystem.Directory.Delete(directory); + } + } + catch + { + // Silent failure - cleanup is opportunistic + } + } +} diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml index 40c13b2c3..e310af129 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml @@ -4,7 +4,17 @@ if (diagram?.EncodedUrl != null) {
- @diagram.DiagramType diagram + @if (!string.IsNullOrEmpty(diagram.LocalSvgPath)) + { + @diagram.DiagramType diagram + } + else + { + @diagram.DiagramType diagram + }
} else diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index b55d5eeec..b2db89370 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -4,6 +4,7 @@ using Elastic.Markdown.Myst.Directives.Diagram; using FluentAssertions; +using System.IO.Abstractions.TestingHelpers; namespace Elastic.Markdown.Tests.Directives; @@ -30,7 +31,22 @@ flowchart LR public void GeneratesEncodedUrl() => Block!.EncodedUrl.Should().StartWith("https://kroki.io/mermaid/svg/"); [Fact] - public void RendersImageTag() => Html.Should().Contain(" Html.Should().Contain(" Block!.ContentHash.Should().NotBeNullOrEmpty(); + + [Fact] + public void GeneratesLocalSvgPath() => Block!.LocalSvgPath.Should().Contain("images/generated-graphs/"); + + [Fact] + public void LocalSvgPathContainsHash() => Block!.LocalSvgPath.Should().Contain(Block!.ContentHash!); + + [Fact] + public void LocalSvgPathContainsDiagramType() => Block!.LocalSvgPath.Should().Contain("-diagram-mermaid-"); + + [Fact] + public void RendersLocalPathWithFallback() => Html.Should().Contain("onerror=\"this.src='https://kroki.io/mermaid/svg/"); } public class DiagramBlockD2Tests(ITestOutputHelper output) : DirectiveTest(output, @@ -82,3 +98,81 @@ public class DiagramBlockEmptyTests(ITestOutputHelper output) : DirectiveTest Collector.Diagnostics.Should().ContainSingle(d => d.Message.Contains("Diagram directive requires content")); } + +public class DiagramRegistryTests +{ + [Fact] + public void ClearResetsRegistry() + { + // Arrange + DiagramRegistry.RegisterDiagram("test-path.svg"); + + // Act + DiagramRegistry.Clear(); + + // Assert - registry should be empty, cleanup should not find any files to remove + var fileSystem = new MockFileSystem(); + var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/test", fileSystem); + cleanedCount.Should().Be(0); + } + + [Fact] + public void CleanupRemovesUnusedFiles() + { + // Arrange + DiagramRegistry.Clear(); + DiagramRegistry.RegisterDiagram("images/generated-graphs/active-diagram.svg"); + + var fileSystem = new MockFileSystem(new Dictionary + { + ["/output/images/generated-graphs/active-diagram.svg"] = new MockFileData("active"), + ["/output/images/generated-graphs/unused-diagram.svg"] = new MockFileData("unused"), + ["/output/images/generated-graphs/another-unused.svg"] = new MockFileData("another") + }); + + // Act + var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/output", fileSystem); + + // Assert + cleanedCount.Should().Be(2); + fileSystem.File.Exists("/output/images/generated-graphs/active-diagram.svg").Should().BeTrue(); + fileSystem.File.Exists("/output/images/generated-graphs/unused-diagram.svg").Should().BeFalse(); + fileSystem.File.Exists("/output/images/generated-graphs/another-unused.svg").Should().BeFalse(); + } + + [Fact] + public void CleanupHandlesMissingDirectory() + { + // Arrange + DiagramRegistry.Clear(); + var fileSystem = new MockFileSystem(); + + // Act & Assert - should not throw + var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/nonexistent", fileSystem); + cleanedCount.Should().Be(0); + } + + [Fact] + public void CleanupRemovesEmptyDirectories() + { + // Arrange + DiagramRegistry.Clear(); + var fileSystem = new MockFileSystem(new Dictionary + { + ["/output/images/generated-graphs/unused.svg"] = new MockFileData("unused") + }); + + // Verify initial state + fileSystem.Directory.Exists("/output/images/generated-graphs").Should().BeTrue(); + fileSystem.File.Exists("/output/images/generated-graphs/unused.svg").Should().BeTrue(); + + // Act + var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/output", fileSystem); + + // Assert + cleanedCount.Should().Be(1); + fileSystem.File.Exists("/output/images/generated-graphs/unused.svg").Should().BeFalse(); + fileSystem.Directory.Exists("/output/images/generated-graphs").Should().BeFalse(); + // Note: /output/images may still exist if MockFileSystem creates it as a parent directory + } +} From 04ac6c1950f6205abb82f26c599eab4731cae026 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 15:31:12 +0200 Subject: [PATCH 02/11] Fix various issues --- docs/_docset.yml | 2 +- docs/syntax/diagrams.md | 2 +- .../Myst/Directives/Diagram/DiagramBlock.cs | 45 ++++++++++++++----- .../Directives/Diagram/DiagramHttpClient.cs | 21 +++++++++ .../Directives/Diagram/DiagramView.cshtml | 2 +- .../docs-builder/Http/DocumentationWebHost.cs | 7 +++ 6 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 src/Elastic.Markdown/Myst/Directives/Diagram/DiagramHttpClient.cs diff --git a/docs/_docset.yml b/docs/_docset.yml index 58475f7ef..dd3fb3441 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -86,7 +86,7 @@ toc: - file: code.md - file: comments.md - file: conditionals.md - - hidden: diagrams.md + - file: diagrams.md - file: dropdowns.md - file: definition-lists.md - file: example_blocks.md diff --git a/docs/syntax/diagrams.md b/docs/syntax/diagrams.md index ce20375de..b4b8ea0b9 100644 --- a/docs/syntax/diagrams.md +++ b/docs/syntax/diagrams.md @@ -84,7 +84,7 @@ sequenceDiagram :::::{tab-item} Rendered ::::{diagram} mermaid sequenceDiagram - participant A as Alice + participant A as Ada participant B as Bob A->>B: Hello Bob, how are you? B-->>A: Great! diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index a85c250da..e9ca745fd 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -71,7 +71,8 @@ public override void FinalizeAndValidate(ParserContext context) // Register diagram for tracking and cleanup DiagramRegistry.RegisterDiagram(LocalSvgPath); - // Cache diagram asynchronously + // Cache diagram asynchronously - fire and forget + // Use simplified approach without lock files to avoid orphaned locks _ = Task.Run(() => TryCacheDiagramAsync(context)); } @@ -125,7 +126,7 @@ private async Task TryCacheDiagramAsync(ParserContext context) var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; var fullPath = Path.Combine(outputDirectory, LocalSvgPath); - // Skip if file already exists + // Skip if file already exists - simple check without locking if (File.Exists(fullPath)) return; @@ -136,17 +137,41 @@ private async Task TryCacheDiagramAsync(ParserContext context) _ = Directory.CreateDirectory(directory); } - // Download SVG from Kroki - using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(30) }; - var svgContent = await httpClient.GetStringAsync(EncodedUrl); + // Download SVG from Kroki using shared HttpClient + var svgContent = await DiagramHttpClient.Instance.GetStringAsync(EncodedUrl); - // Write to local file - await File.WriteAllTextAsync(fullPath, svgContent); + // Basic validation - ensure we got SVG content + // SVG can start with XML declaration, DOCTYPE, or directly with + if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains(" +/// Shared HttpClient for diagram downloads to avoid resource exhaustion +/// +public static class DiagramHttpClient +{ + private static readonly Lazy LazyHttpClient = new(() => new HttpClient + { + Timeout = TimeSpan.FromSeconds(30) + }); + + /// + /// Shared HttpClient instance for diagram downloads + /// + public static HttpClient Instance => LazyHttpClient.Value; +} diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml index e310af129..1b3433798 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml @@ -6,7 +6,7 @@
@if (!string.IsNullOrEmpty(diagram.LocalSvgPath)) { - @diagram.DiagramType diagram diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index 1f84566ba..884d0ccb3 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -18,6 +18,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Westwind.AspNetCore.LiveReload; @@ -152,6 +153,12 @@ await context.Response.WriteAsync(@" FileProvider = new EmbeddedOrPhysicalFileProvider(Context), RequestPath = "/_static" }) + .UseStaticFiles( + new StaticFileOptions + { + FileProvider = new PhysicalFileProvider(Context.DocumentationOutputDirectory.FullName), + RequestPath = "" + }) .UseRouting(); _ = _webApplication.MapGet("/", (ReloadableGeneratorState holder, Cancel ctx) => From 1896b71f84b8638676168edeff331f4193dc5933 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 15:46:34 +0200 Subject: [PATCH 03/11] Fix lint errors --- tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index b2db89370..30aa0b4fa 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -2,9 +2,9 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Abstractions.TestingHelpers; using Elastic.Markdown.Myst.Directives.Diagram; using FluentAssertions; -using System.IO.Abstractions.TestingHelpers; namespace Elastic.Markdown.Tests.Directives; From 1383eb0e792e7f4b2ce0756cd005b1bc5de827cb Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 15:47:59 +0200 Subject: [PATCH 04/11] Add callout --- docs/syntax/diagrams.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/syntax/diagrams.md b/docs/syntax/diagrams.md index b4b8ea0b9..792dcf216 100644 --- a/docs/syntax/diagrams.md +++ b/docs/syntax/diagrams.md @@ -2,6 +2,10 @@ The `diagram` directive allows you to render various types of diagrams using the [Kroki](https://kroki.io/) service. Kroki supports many diagram types including Mermaid, D2, Graphviz, PlantUML, and more. +::::{warning} +This is an experimental feature. It may change in the future. +:::: + ## Basic usage The basic syntax for the diagram directive is: From 8d0e54319c4401b6fd7025e793fc50c01078d449 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 15:52:19 +0200 Subject: [PATCH 05/11] Fix test --- tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index 30aa0b4fa..054cfb9c1 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -31,7 +31,7 @@ flowchart LR public void GeneratesEncodedUrl() => Block!.EncodedUrl.Should().StartWith("https://kroki.io/mermaid/svg/"); [Fact] - public void RendersImageTag() => Html.Should().Contain(" Html.Should().Contain(" Block!.ContentHash.Should().NotBeNullOrEmpty(); From b205ac25e023a1ed557b5cee1f6ebca2e7518f6e Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 16:10:23 +0200 Subject: [PATCH 06/11] Fix for slashes (hopefully) --- src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index e9ca745fd..776c48bc7 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -112,7 +112,10 @@ private string GenerateLocalPath(ParserContext context) } var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg"; - return Path.Combine("images", "generated-graphs", filename); + var localPath = Path.Combine("images", "generated-graphs", filename); + + // Normalize path separators to forward slashes for web compatibility + return localPath.Replace(Path.DirectorySeparatorChar, '/'); } private async Task TryCacheDiagramAsync(ParserContext context) From 5eae0cde720195ecf6fc56208557c592ed43bcc6 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Tue, 22 Jul 2025 19:01:52 +0200 Subject: [PATCH 07/11] Tech review changes --- .../BuildContext.cs | 4 + .../Diagram/DiagramRegistry.cs | 209 ++++++++++++++++++ .../DocumentationGenerator.cs | 12 +- .../Myst/Directives/Diagram/DiagramBlock.cs | 72 +----- .../Directives/Diagram/DiagramHttpClient.cs | 21 -- .../Directives/Diagram/DiagramRegistry.cs | 146 ------------ src/Elastic.Markdown/Myst/ParserContext.cs | 2 + .../Directives/DiagramTests.cs | 84 +++---- 8 files changed, 267 insertions(+), 283 deletions(-) create mode 100644 src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs delete mode 100644 src/Elastic.Markdown/Myst/Directives/Diagram/DiagramHttpClient.cs delete mode 100644 src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs diff --git a/src/Elastic.Documentation.Configuration/BuildContext.cs b/src/Elastic.Documentation.Configuration/BuildContext.cs index 154d90b79..9b1134462 100644 --- a/src/Elastic.Documentation.Configuration/BuildContext.cs +++ b/src/Elastic.Documentation.Configuration/BuildContext.cs @@ -6,6 +6,7 @@ using System.Reflection; using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Configuration.Versions; using Elastic.Documentation.Diagnostics; @@ -64,6 +65,8 @@ public string? UrlPathPrefix init => _urlPathPrefix = value; } + public DiagramRegistry DiagramRegistry { get; } + public BuildContext(IDiagnosticsCollector collector, IFileSystem fileSystem, VersionsConfiguration versionsConfig) : this(collector, fileSystem, fileSystem, versionsConfig, null, null) { @@ -105,5 +108,6 @@ public BuildContext( { Enabled = false }; + DiagramRegistry = new DiagramRegistry(writeFileSystem); } } diff --git a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs new file mode 100644 index 000000000..31ccf47f8 --- /dev/null +++ b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs @@ -0,0 +1,209 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Concurrent; +using System.IO.Abstractions; +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation.Configuration.Diagram; + +/// +/// Information about a diagram that needs to be cached +/// +/// Local SVG path relative to output directory +/// Encoded Kroki URL for downloading +/// Full path to output directory +public record DiagramCacheInfo(string LocalSvgPath, string EncodedUrl, string OutputDirectory); + +/// +/// Registry to track active diagrams and manage cleanup of outdated cached files +/// +/// File system for write/delete operations during cleanup +public class DiagramRegistry(IFileSystem writeFileSystem) : IDisposable +{ + private readonly ConcurrentDictionary _activeDiagrams = new(); + private readonly ConcurrentDictionary _diagramsToCache = new(); + private readonly IFileSystem _writeFileSystem = writeFileSystem; + private readonly HttpClient _httpClient = new() { Timeout = TimeSpan.FromSeconds(30) }; + + /// + /// Register a diagram for caching (collects info for later batch processing) + /// + /// The local SVG path relative to output directory + /// The encoded Kroki URL for downloading + /// The full path to output directory + public void RegisterDiagramForCaching(string localSvgPath, string encodedUrl, string outputDirectory) + { + if (string.IsNullOrEmpty(localSvgPath) || string.IsNullOrEmpty(encodedUrl)) + return; + + _ = _activeDiagrams.TryAdd(localSvgPath, true); + _ = _diagramsToCache.TryAdd(localSvgPath, new DiagramCacheInfo(localSvgPath, encodedUrl, outputDirectory)); + } + + /// + /// Clear all registered diagrams (called at start of build) + /// + public void Clear() + { + _activeDiagrams.Clear(); + _diagramsToCache.Clear(); + } + + /// + /// Create cached diagram files by downloading from Kroki in parallel + /// + /// Logger for reporting download activity + /// File system for checking existing files + /// Number of diagrams downloaded + public async Task CreateDiagramCachedFiles(ILogger logger, IFileSystem readFileSystem) + { + if (_diagramsToCache.IsEmpty) + return 0; + + var downloadCount = 0; + + await Parallel.ForEachAsync(_diagramsToCache.Values, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = CancellationToken.None + }, async (diagramInfo, ct) => + { + try + { + var fullPath = _writeFileSystem.Path.Combine(diagramInfo.OutputDirectory, diagramInfo.LocalSvgPath); + + // Skip if file already exists + if (readFileSystem.File.Exists(fullPath)) + return; + + // Create directory if needed + var directory = _writeFileSystem.Path.GetDirectoryName(fullPath); + if (directory != null && !_writeFileSystem.Directory.Exists(directory)) + { + _ = _writeFileSystem.Directory.CreateDirectory(directory); + } + + // Download SVG content + var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct); + + // Validate SVG content + if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains(" 0) + { + logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount); + } + + return downloadCount; + } + + /// + /// Clean up unused diagram files from the cache directory + /// + /// The output directory containing cached diagrams + /// Number of files cleaned up + public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory) + { + var graphsDir = _writeFileSystem.Path.Combine(outputDirectory.FullName, "images", "generated-graphs"); + if (!_writeFileSystem.Directory.Exists(graphsDir)) + return 0; + + var existingFiles = _writeFileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories); + var cleanedCount = 0; + + try + { + foreach (var file in existingFiles) + { + var relativePath = _writeFileSystem.Path.GetRelativePath(outputDirectory.FullName, file); + var normalizedPath = relativePath.Replace(_writeFileSystem.Path.DirectorySeparatorChar, '/'); + + if (!_activeDiagrams.ContainsKey(normalizedPath)) + { + try + { + _writeFileSystem.File.Delete(file); + cleanedCount++; + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + } + + // Clean up empty directories + CleanupEmptyDirectories(graphsDir); + } + catch + { + // Silent failure - cleanup is opportunistic + } + + return cleanedCount; + } + + private void CleanupEmptyDirectories(string directory) + { + try + { + foreach (var subDir in _writeFileSystem.Directory.GetDirectories(directory)) + { + CleanupEmptyDirectories(subDir); + + if (!_writeFileSystem.Directory.EnumerateFileSystemEntries(subDir).Any()) + { + try + { + _writeFileSystem.Directory.Delete(subDir); + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + } + } + catch + { + // Silent failure - cleanup is opportunistic + } + } + + /// + /// Dispose of resources, including the HttpClient + /// + public void Dispose() + { + _httpClient.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 3a89faefc..8ff994888 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -6,6 +6,7 @@ using System.Text.Json; using Elastic.Documentation; using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Legacy; using Elastic.Documentation.Links; using Elastic.Documentation.Serialization; @@ -108,7 +109,7 @@ public async Task ResolveDirectoryTree(Cancel ctx) public async Task GenerateAll(Cancel ctx) { // Clear diagram registry for fresh tracking - DiagramRegistry.Clear(); + DocumentationSet.Context.DiagramRegistry.Clear(); var result = new GenerationResult(); @@ -146,8 +147,15 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating links.json"); var linkReference = await GenerateLinkReference(ctx); + // Download diagram files in parallel + var downloadedCount = await DocumentationSet.Context.DiagramRegistry.CreateDiagramCachedFiles(_logger, DocumentationSet.Context.ReadFileSystem); + if (downloadedCount > 0) + { + _logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount); + } + // Clean up unused diagram files - var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory.FullName); + var cleanedCount = DocumentationSet.Context.DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory); if (cleanedCount > 0) { _logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount); diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index 776c48bc7..de70ba456 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -4,6 +4,7 @@ using System.Security.Cryptography; using System.Text; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Diagnostics; namespace Elastic.Markdown.Myst.Directives.Diagram; @@ -68,12 +69,9 @@ public override void FinalizeAndValidate(ParserContext context) return; } - // Register diagram for tracking and cleanup - DiagramRegistry.RegisterDiagram(LocalSvgPath); - - // Cache diagram asynchronously - fire and forget - // Use simplified approach without lock files to avoid orphaned locks - _ = Task.Run(() => TryCacheDiagramAsync(context)); + // Register diagram for tracking, cleanup, and batch caching + var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; + context.DiagramRegistry.RegisterDiagramForCaching(LocalSvgPath, EncodedUrl, outputDirectory); } private string? ExtractContent() @@ -106,9 +104,9 @@ private string GenerateContentHash(string diagramType, string content) private string GenerateLocalPath(ParserContext context) { var markdownFileName = "unknown"; - if (context.MarkdownSourcePath?.FullName != null) + if (context.MarkdownSourcePath?.Name is not null) { - markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.FullName); + markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.Name); } var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg"; @@ -118,63 +116,5 @@ private string GenerateLocalPath(ParserContext context) return localPath.Replace(Path.DirectorySeparatorChar, '/'); } - private async Task TryCacheDiagramAsync(ParserContext context) - { - if (string.IsNullOrEmpty(EncodedUrl) || string.IsNullOrEmpty(LocalSvgPath)) - return; - - try - { - // Determine the full output path - var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; - var fullPath = Path.Combine(outputDirectory, LocalSvgPath); - - // Skip if file already exists - simple check without locking - if (File.Exists(fullPath)) - return; - - // Create directory if it doesn't exist - var directory = Path.GetDirectoryName(fullPath); - if (directory != null && !Directory.Exists(directory)) - { - _ = Directory.CreateDirectory(directory); - } - - // Download SVG from Kroki using shared HttpClient - var svgContent = await DiagramHttpClient.Instance.GetStringAsync(EncodedUrl); - - // Basic validation - ensure we got SVG content - // SVG can start with XML declaration, DOCTYPE, or directly with - if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains(" -/// Shared HttpClient for diagram downloads to avoid resource exhaustion -/// -public static class DiagramHttpClient -{ - private static readonly Lazy LazyHttpClient = new(() => new HttpClient - { - Timeout = TimeSpan.FromSeconds(30) - }); - - /// - /// Shared HttpClient instance for diagram downloads - /// - public static HttpClient Instance => LazyHttpClient.Value; -} diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs deleted file mode 100644 index 351c6a9bd..000000000 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramRegistry.cs +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.IO.Abstractions; - -namespace Elastic.Markdown.Myst.Directives.Diagram; - -/// -/// Registry to track active diagrams and manage cleanup of outdated cached files -/// -public static class DiagramRegistry -{ - private static readonly HashSet ActiveDiagrams = []; - private static readonly Lock Lock = new(); - - /// - /// Register a diagram as active during the current build - /// - /// The local SVG path relative to output directory - public static void RegisterDiagram(string localSvgPath) - { - if (string.IsNullOrEmpty(localSvgPath)) - return; - - lock (Lock) - { - _ = ActiveDiagrams.Add(localSvgPath); - } - } - - /// - /// Get all currently registered active diagrams - /// - /// Collection of active diagram paths - public static IReadOnlyCollection GetActiveDiagrams() - { - lock (Lock) - { - return ActiveDiagrams.ToArray(); - } - } - - /// - /// Clear all registered diagrams (typically called at start of build) - /// - public static void Clear() - { - lock (Lock) - { - ActiveDiagrams.Clear(); - } - } - - /// - /// Clean up unused diagram files from the output directory - /// - /// The output directory path - /// Number of files cleaned up - public static int CleanupUnusedDiagrams(string outputDirectory) => - CleanupUnusedDiagrams(outputDirectory, new FileSystem()); - - /// - /// Clean up unused diagram files from the output directory - /// - /// The output directory path - /// File system abstraction for testing - /// Number of files cleaned up - public static int CleanupUnusedDiagrams(string outputDirectory, IFileSystem fileSystem) - { - if (string.IsNullOrEmpty(outputDirectory)) - return 0; - - var graphsDir = fileSystem.Path.Combine(outputDirectory, "images", "generated-graphs"); - if (!fileSystem.Directory.Exists(graphsDir)) - return 0; - - var cleanedCount = 0; - var activePaths = GetActiveDiagrams(); - - try - { - var existingFiles = fileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories); - - foreach (var file in existingFiles) - { - var relativePath = fileSystem.Path.GetRelativePath(outputDirectory, file); - - // Convert to forward slashes for consistent comparison - var normalizedPath = relativePath.Replace(fileSystem.Path.DirectorySeparatorChar, '/'); - - if (!activePaths.Any(active => active.Replace(fileSystem.Path.DirectorySeparatorChar, '/') == normalizedPath)) - { - try - { - fileSystem.File.Delete(file); - cleanedCount++; - } - catch - { - // Silent failure - cleanup is opportunistic - } - } - } - - // Clean up empty directories - CleanupEmptyDirectories(graphsDir, fileSystem); - } - catch - { - // Silent failure - cleanup is opportunistic - } - - return cleanedCount; - } - - /// - /// Remove empty directories recursively - /// - /// Directory to clean up - /// File system abstraction - private static void CleanupEmptyDirectories(string directory, IFileSystem fileSystem) - { - try - { - if (!fileSystem.Directory.Exists(directory)) - return; - - // Clean up subdirectories first - foreach (var subDir in fileSystem.Directory.GetDirectories(directory)) - { - CleanupEmptyDirectories(subDir, fileSystem); - } - - // Remove directory if it's empty - if (!fileSystem.Directory.EnumerateFileSystemEntries(directory).Any()) - { - fileSystem.Directory.Delete(directory); - } - } - catch - { - // Silent failure - cleanup is opportunistic - } - } -} diff --git a/src/Elastic.Markdown/Myst/ParserContext.cs b/src/Elastic.Markdown/Myst/ParserContext.cs index 03fc703ad..0ab2e4046 100644 --- a/src/Elastic.Markdown/Myst/ParserContext.cs +++ b/src/Elastic.Markdown/Myst/ParserContext.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; @@ -58,6 +59,7 @@ public class ParserContext : MarkdownParserContext, IParserResolvers public string CurrentUrlPath { get; } public YamlFrontMatter? YamlFrontMatter { get; } public BuildContext Build { get; } + public DiagramRegistry DiagramRegistry => Build.DiagramRegistry; public bool SkipValidation { get; } public Func DocumentationFileLookup { get; } public IReadOnlyDictionary Substitutions { get; } diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index 054cfb9c1..734f6a5a4 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions.TestingHelpers; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Myst.Directives.Diagram; using FluentAssertions; @@ -102,77 +103,64 @@ public void EmptyContentGeneratesError() => public class DiagramRegistryTests { [Fact] - public void ClearResetsRegistry() + public void CleanupUnusedDiagramsWithNoActiveFilesCleansAllFiles() { - // Arrange - DiagramRegistry.RegisterDiagram("test-path.svg"); + var fileSystem = new MockFileSystem(); + var registry = new DiagramRegistry(fileSystem); + registry.RegisterDiagramForCaching("test-path.svg", "http://example.com/test", "/test"); - // Act - DiagramRegistry.Clear(); + // Clear registry to simulate no active diagrams + registry.Clear(); - // Assert - registry should be empty, cleanup should not find any files to remove - var fileSystem = new MockFileSystem(); - var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/test", fileSystem); - cleanedCount.Should().Be(0); + var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/test")); + + cleanedCount.Should().Be(0); // No files to clean since directory doesn't exist } [Fact] - public void CleanupRemovesUnusedFiles() + public void CleanupUnusedDiagramsWithActiveAndUnusedFilesCleansOnlyUnused() { - // Arrange - DiagramRegistry.Clear(); - DiagramRegistry.RegisterDiagram("images/generated-graphs/active-diagram.svg"); - - var fileSystem = new MockFileSystem(new Dictionary - { - ["/output/images/generated-graphs/active-diagram.svg"] = new MockFileData("active"), - ["/output/images/generated-graphs/unused-diagram.svg"] = new MockFileData("unused"), - ["/output/images/generated-graphs/another-unused.svg"] = new MockFileData("another") - }); - - // Act - var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/output", fileSystem); - - // Assert - cleanedCount.Should().Be(2); + var fileSystem = new MockFileSystem(); + var registry = new DiagramRegistry(fileSystem); + registry.Clear(); + registry.RegisterDiagramForCaching("images/generated-graphs/active-diagram.svg", "http://example.com/active", "/output"); + + fileSystem.AddDirectory("/output/images/generated-graphs"); + fileSystem.AddFile("/output/images/generated-graphs/active-diagram.svg", "active content"); + fileSystem.AddFile("/output/images/generated-graphs/unused-diagram.svg", "unused content"); + + var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/output")); + + cleanedCount.Should().Be(1); fileSystem.File.Exists("/output/images/generated-graphs/active-diagram.svg").Should().BeTrue(); fileSystem.File.Exists("/output/images/generated-graphs/unused-diagram.svg").Should().BeFalse(); - fileSystem.File.Exists("/output/images/generated-graphs/another-unused.svg").Should().BeFalse(); } [Fact] - public void CleanupHandlesMissingDirectory() + public void CleanupUnusedDiagramsWithNonexistentDirectoryReturnsZero() { - // Arrange - DiagramRegistry.Clear(); var fileSystem = new MockFileSystem(); + var registry = new DiagramRegistry(fileSystem); + registry.Clear(); + + var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/nonexistent")); - // Act & Assert - should not throw - var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/nonexistent", fileSystem); cleanedCount.Should().Be(0); } [Fact] - public void CleanupRemovesEmptyDirectories() + public void CleanupUnusedDiagramsRemovesEmptyDirectories() { - // Arrange - DiagramRegistry.Clear(); - var fileSystem = new MockFileSystem(new Dictionary - { - ["/output/images/generated-graphs/unused.svg"] = new MockFileData("unused") - }); + var fileSystem = new MockFileSystem(); + var registry = new DiagramRegistry(fileSystem); + registry.Clear(); - // Verify initial state - fileSystem.Directory.Exists("/output/images/generated-graphs").Should().BeTrue(); - fileSystem.File.Exists("/output/images/generated-graphs/unused.svg").Should().BeTrue(); + fileSystem.AddDirectory("/output/images/generated-graphs/subdir"); + fileSystem.AddFile("/output/images/generated-graphs/subdir/unused.svg", "content"); - // Act - var cleanedCount = DiagramRegistry.CleanupUnusedDiagrams("/output", fileSystem); + var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/output")); - // Assert cleanedCount.Should().Be(1); - fileSystem.File.Exists("/output/images/generated-graphs/unused.svg").Should().BeFalse(); - fileSystem.Directory.Exists("/output/images/generated-graphs").Should().BeFalse(); - // Note: /output/images may still exist if MockFileSystem creates it as a parent directory + fileSystem.Directory.Exists("/output/images/generated-graphs/subdir").Should().BeFalse(); } } From d8e64299c52fc66c326814cdd6b70e5026169890 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferri Benedetti Date: Wed, 23 Jul 2025 11:52:13 +0200 Subject: [PATCH 08/11] Edits --- .../DocumentationGenerator.cs | 29 ++++++++++++++----- .../Myst/Directives/Diagram/DiagramBlock.cs | 6 ++++ .../docs-builder/Http/DocumentationWebHost.cs | 17 +++++++---- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 8ff994888..cde7bd8a1 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -147,25 +147,38 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating links.json"); var linkReference = await GenerateLinkReference(ctx); - // Download diagram files in parallel + await CreateDiagramCachedFiles(); + CleanupUnusedDiagrams(); + + // ReSharper disable once WithExpressionModifiesAllMembers + return result with + { + Redirects = linkReference.Redirects ?? [] + }; + } + + /// + /// Downloads diagram files in parallel from Kroki + /// + public async Task CreateDiagramCachedFiles() + { var downloadedCount = await DocumentationSet.Context.DiagramRegistry.CreateDiagramCachedFiles(_logger, DocumentationSet.Context.ReadFileSystem); if (downloadedCount > 0) { _logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount); } + } - // Clean up unused diagram files + /// + /// Cleans up unused diagram files from the output directory + /// + public void CleanupUnusedDiagrams() + { var cleanedCount = DocumentationSet.Context.DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory); if (cleanedCount > 0) { _logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount); } - - // ReSharper disable once WithExpressionModifiesAllMembers - return result with - { - Redirects = linkReference.Redirects ?? [] - }; } private async Task ProcessDocumentationFiles(HashSet offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx) diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index de70ba456..a70f8db94 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -6,6 +6,7 @@ using System.Text; using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; namespace Elastic.Markdown.Myst.Directives.Diagram; @@ -70,7 +71,12 @@ public override void FinalizeAndValidate(ParserContext context) } // Register diagram for tracking, cleanup, and batch caching + // Use the markdown file's scope directory for proper relative path resolution var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; + if (context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown) + { + outputDirectory = currentMarkdown.ScopeDirectory.FullName; + } context.DiagramRegistry.RegisterDiagramForCaching(LocalSvgPath, EncodedUrl, outputDirectory); } diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index 884d0ccb3..5e5ee7e70 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO; using System.IO.Abstractions; using System.Net; using System.Runtime.InteropServices; @@ -153,12 +154,6 @@ await context.Response.WriteAsync(@" FileProvider = new EmbeddedOrPhysicalFileProvider(Context), RequestPath = "/_static" }) - .UseStaticFiles( - new StaticFileOptions - { - FileProvider = new PhysicalFileProvider(Context.DocumentationOutputDirectory.FullName), - RequestPath = "" - }) .UseRouting(); _ = _webApplication.MapGet("/", (ReloadableGeneratorState holder, Cancel ctx) => @@ -258,6 +253,16 @@ private static async Task ServeDocumentationFile(ReloadableGeneratorSta if (s == "index.md") return Results.Redirect(generator.DocumentationSet.MarkdownFiles.First().Url); + // Check for cached SVG files (e.g., generated diagrams) in the output directory + if (Path.GetExtension(slug).Equals(".svg", StringComparison.OrdinalIgnoreCase)) + { + var svgPath = Path.Combine(generator.DocumentationSet.OutputDirectory.FullName, slug.TrimStart('/')); + if (File.Exists(svgPath)) + { + return Results.File(svgPath, "image/svg+xml"); + } + } + if (!generator.DocumentationSet.FlatMappedFiles.TryGetValue("404.md", out var notFoundDocumentationFile)) return Results.NotFound(); From 4e3a416c6707782de3d3d79c5b2be3bf2d06b531 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 24 Jul 2025 14:34:19 +0200 Subject: [PATCH 09/11] Move registry to documentation set --- Directory.Packages.props | 1 + .../BuildContext.cs | 3 - .../Diagram/DiagramRegistry.cs | 140 +++++++++--------- .../Extensions/IFileInfoExtensions.cs | 25 ++++ .../DocumentationGenerator.cs | 13 +- .../DetectionRules/DetectionRuleFile.cs | 8 +- src/Elastic.Markdown/HtmlWriter.cs | 2 +- src/Elastic.Markdown/IO/DocumentationSet.cs | 6 +- .../Myst/Directives/Diagram/DiagramBlock.cs | 29 ++-- .../Directives/Diagram/DiagramView.cshtml | 4 +- .../Myst/Directives/DirectiveHtmlRenderer.cs | 22 ++- src/Elastic.Markdown/Myst/MarkdownParser.cs | 46 +++--- src/Elastic.Markdown/Myst/ParserContext.cs | 4 +- .../LlmMarkdown/LlmBlockRenderers.cs | 12 +- .../Directives/DiagramTests.cs | 80 +++++----- .../Elastic.Markdown.Tests.csproj | 1 + 16 files changed, 217 insertions(+), 179 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index da5e2015f..6beddb763 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,6 +16,7 @@ + diff --git a/src/Elastic.Documentation.Configuration/BuildContext.cs b/src/Elastic.Documentation.Configuration/BuildContext.cs index 9b1134462..22479014a 100644 --- a/src/Elastic.Documentation.Configuration/BuildContext.cs +++ b/src/Elastic.Documentation.Configuration/BuildContext.cs @@ -65,8 +65,6 @@ public string? UrlPathPrefix init => _urlPathPrefix = value; } - public DiagramRegistry DiagramRegistry { get; } - public BuildContext(IDiagnosticsCollector collector, IFileSystem fileSystem, VersionsConfiguration versionsConfig) : this(collector, fileSystem, fileSystem, versionsConfig, null, null) { @@ -108,6 +106,5 @@ public BuildContext( { Enabled = false }; - DiagramRegistry = new DiagramRegistry(writeFileSystem); } } diff --git a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs index 31ccf47f8..e971d4c7f 100644 --- a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs +++ b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.IO.Abstractions; +using Elastic.Documentation.Extensions; using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Configuration.Diagram; @@ -11,53 +12,41 @@ namespace Elastic.Documentation.Configuration.Diagram; /// /// Information about a diagram that needs to be cached /// -/// Local SVG path relative to output directory +/// The intended cache output file location /// Encoded Kroki URL for downloading -/// Full path to output directory -public record DiagramCacheInfo(string LocalSvgPath, string EncodedUrl, string OutputDirectory); +public record DiagramCacheInfo(IFileInfo OutputFile, string EncodedUrl); -/// /// Registry to track active diagrams and manage cleanup of outdated cached files -/// -/// File system for write/delete operations during cleanup -public class DiagramRegistry(IFileSystem writeFileSystem) : IDisposable +public class DiagramRegistry(ILoggerFactory logFactory, BuildContext context) : IDisposable { - private readonly ConcurrentDictionary _activeDiagrams = new(); + private readonly ILogger _logger = logFactory.CreateLogger(); private readonly ConcurrentDictionary _diagramsToCache = new(); - private readonly IFileSystem _writeFileSystem = writeFileSystem; + private readonly IFileSystem _writeFileSystem = context.WriteFileSystem; + private readonly IFileSystem _readFileSystem = context.ReadFileSystem; private readonly HttpClient _httpClient = new() { Timeout = TimeSpan.FromSeconds(30) }; /// /// Register a diagram for caching (collects info for later batch processing) /// - /// The local SVG path relative to output directory + /// The local SVG path relative to the output directory /// The encoded Kroki URL for downloading - /// The full path to output directory - public void RegisterDiagramForCaching(string localSvgPath, string encodedUrl, string outputDirectory) + /// The full path to the output directory + public void RegisterDiagramForCaching(IFileInfo outputFile, string encodedUrl) { - if (string.IsNullOrEmpty(localSvgPath) || string.IsNullOrEmpty(encodedUrl)) + if (string.IsNullOrEmpty(encodedUrl)) return; - _ = _activeDiagrams.TryAdd(localSvgPath, true); - _ = _diagramsToCache.TryAdd(localSvgPath, new DiagramCacheInfo(localSvgPath, encodedUrl, outputDirectory)); - } + if (!outputFile.IsSubPathOf(context.DocumentationOutputDirectory)) + return; - /// - /// Clear all registered diagrams (called at start of build) - /// - public void Clear() - { - _activeDiagrams.Clear(); - _diagramsToCache.Clear(); + _ = _diagramsToCache.TryAdd(outputFile.FullName, new DiagramCacheInfo(outputFile, encodedUrl)); } /// /// Create cached diagram files by downloading from Kroki in parallel /// - /// Logger for reporting download activity - /// File system for checking existing files /// Number of diagrams downloaded - public async Task CreateDiagramCachedFiles(ILogger logger, IFileSystem readFileSystem) + public async Task CreateDiagramCachedFiles(Cancel ctx) { if (_diagramsToCache.IsEmpty) return 0; @@ -67,23 +56,24 @@ public async Task CreateDiagramCachedFiles(ILogger logger, IFileSystem read await Parallel.ForEachAsync(_diagramsToCache.Values, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, - CancellationToken = CancellationToken.None + CancellationToken = ctx }, async (diagramInfo, ct) => { + var localPath = _readFileSystem.Path.GetRelativePath(context.DocumentationOutputDirectory.FullName, diagramInfo.OutputFile.FullName); + try { - var fullPath = _writeFileSystem.Path.Combine(diagramInfo.OutputDirectory, diagramInfo.LocalSvgPath); + if (!diagramInfo.OutputFile.IsSubPathOf(context.DocumentationOutputDirectory)) + return; - // Skip if file already exists - if (readFileSystem.File.Exists(fullPath)) + // Skip if the file already exists + if (_readFileSystem.File.Exists(diagramInfo.OutputFile.FullName)) return; - // Create directory if needed - var directory = _writeFileSystem.Path.GetDirectoryName(fullPath); + // Create the directory if needed + var directory = _writeFileSystem.Path.GetDirectoryName(diagramInfo.OutputFile.FullName); if (directory != null && !_writeFileSystem.Directory.Exists(directory)) - { _ = _writeFileSystem.Directory.CreateDirectory(directory); - } // Download SVG content var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct); @@ -91,36 +81,34 @@ public async Task CreateDiagramCachedFiles(ILogger logger, IFileSystem read // Validate SVG content if (string.IsNullOrWhiteSpace(svgContent) || !svgContent.Contains(" 0) - { - logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount); - } + _logger.LogInformation("Downloaded {DownloadCount} diagram files from Kroki", downloadCount); return downloadCount; } @@ -128,26 +116,27 @@ public async Task CreateDiagramCachedFiles(ILogger logger, IFileSystem read /// /// Clean up unused diagram files from the cache directory /// - /// The output directory containing cached diagrams /// Number of files cleaned up - public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory) + public int CleanupUnusedDiagrams() { - var graphsDir = _writeFileSystem.Path.Combine(outputDirectory.FullName, "images", "generated-graphs"); - if (!_writeFileSystem.Directory.Exists(graphsDir)) + if (!_readFileSystem.Directory.Exists(context.DocumentationOutputDirectory.FullName)) + return 0; + var folders = _writeFileSystem.Directory.GetDirectories(context.DocumentationOutputDirectory.FullName, "generated-graphs", SearchOption.AllDirectories); + var existingFiles = folders + .Select(f => (Folder: f, Files: _writeFileSystem.Directory.GetFiles(f, "*.svg", SearchOption.TopDirectoryOnly))) + .ToArray(); + if (existingFiles.Length == 0) return 0; - - var existingFiles = _writeFileSystem.Directory.GetFiles(graphsDir, "*.svg", SearchOption.AllDirectories); var cleanedCount = 0; try { - foreach (var file in existingFiles) + foreach (var (folder, files) in existingFiles) { - var relativePath = _writeFileSystem.Path.GetRelativePath(outputDirectory.FullName, file); - var normalizedPath = relativePath.Replace(_writeFileSystem.Path.DirectorySeparatorChar, '/'); - - if (!_activeDiagrams.ContainsKey(normalizedPath)) + foreach (var file in files) { + if (_diagramsToCache.ContainsKey(file)) + continue; try { _writeFileSystem.File.Delete(file); @@ -158,10 +147,9 @@ public int CleanupUnusedDiagrams(IDirectoryInfo outputDirectory) // Silent failure - cleanup is opportunistic } } + // Clean up empty directories + CleanupEmptyDirectories(folder); } - - // Clean up empty directories - CleanupEmptyDirectories(graphsDir); } catch { @@ -175,22 +163,26 @@ private void CleanupEmptyDirectories(string directory) { try { - foreach (var subDir in _writeFileSystem.Directory.GetDirectories(directory)) - { - CleanupEmptyDirectories(subDir); + var folder = _writeFileSystem.DirectoryInfo.New(directory); + if (!folder.IsSubPathOf(context.DocumentationOutputDirectory)) + return; - if (!_writeFileSystem.Directory.EnumerateFileSystemEntries(subDir).Any()) - { - try - { - _writeFileSystem.Directory.Delete(subDir); - } - catch - { - // Silent failure - cleanup is opportunistic - } - } - } + if (folder.Name != "generated-graphs") + return; + + if (_writeFileSystem.Directory.EnumerateFileSystemEntries(folder.FullName).Any()) + return; + + _writeFileSystem.Directory.Delete(folder.FullName); + + var parentFolder = folder.Parent; + if (parentFolder is null || parentFolder.Name != "images") + return; + + if (_writeFileSystem.Directory.EnumerateFileSystemEntries(parentFolder.FullName).Any()) + return; + + _writeFileSystem.Directory.Delete(folder.FullName); } catch { diff --git a/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs b/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs index 2a80ef6cf..7534030dc 100644 --- a/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs +++ b/src/Elastic.Documentation/Extensions/IFileInfoExtensions.cs @@ -18,4 +18,29 @@ public static string ReadToEnd(this IFileInfo fileInfo) using var reader = new StreamReader(stream); return reader.ReadToEnd(); } + + /// Validates is in a subdirectory of + public static bool IsSubPathOf(this IFileInfo file, IDirectoryInfo parentDirectory) + { + var parent = file.Directory; + return parent is not null && parent.IsSubPathOf(parentDirectory); + } +} + +public static class IDirectoryInfoExtensions +{ + /// Validates is subdirectory of + public static bool IsSubPathOf(this IDirectoryInfo directory, IDirectoryInfo parentDirectory) + { + var parent = directory; + do + { + if (parent.FullName == parentDirectory.FullName) + return true; + parent = parent.Parent; + } + while (parent != null); + + return false; + } } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index cde7bd8a1..ec6355c1f 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -108,9 +108,6 @@ public async Task ResolveDirectoryTree(Cancel ctx) public async Task GenerateAll(Cancel ctx) { - // Clear diagram registry for fresh tracking - DocumentationSet.Context.DiagramRegistry.Clear(); - var result = new GenerationResult(); var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState(); @@ -147,7 +144,7 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating links.json"); var linkReference = await GenerateLinkReference(ctx); - await CreateDiagramCachedFiles(); + await CreateDiagramCachedFiles(ctx); CleanupUnusedDiagrams(); // ReSharper disable once WithExpressionModifiesAllMembers @@ -160,9 +157,9 @@ public async Task GenerateAll(Cancel ctx) /// /// Downloads diagram files in parallel from Kroki /// - public async Task CreateDiagramCachedFiles() + public async Task CreateDiagramCachedFiles(Cancel ctx) { - var downloadedCount = await DocumentationSet.Context.DiagramRegistry.CreateDiagramCachedFiles(_logger, DocumentationSet.Context.ReadFileSystem); + var downloadedCount = await DocumentationSet.DiagramRegistry.CreateDiagramCachedFiles(ctx); if (downloadedCount > 0) { _logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount); @@ -174,11 +171,9 @@ public async Task CreateDiagramCachedFiles() /// public void CleanupUnusedDiagrams() { - var cleanedCount = DocumentationSet.Context.DiagramRegistry.CleanupUnusedDiagrams(DocumentationSet.OutputDirectory); + var cleanedCount = DocumentationSet.DiagramRegistry.CleanupUnusedDiagrams(); if (cleanedCount > 0) - { _logger.LogInformation("Cleaned up {CleanedCount} unused diagram files", cleanedCount); - } } private async Task ProcessDocumentationFiles(HashSet offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs index df2a9fb57..f1be56972 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs @@ -29,14 +29,14 @@ protected override Task GetMinimalParseDocumentAsync(Cancel ct { Title = "Prebuilt detection rules reference"; var markdown = GetMarkdown(); - var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null); + var document = MarkdownParser.MinimalParseString(markdown, SourceFile, null); return Task.FromResult(document); } protected override Task GetParseDocumentAsync(Cancel ctx) { var markdown = GetMarkdown(); - var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null); + var document = MarkdownParser.ParseString(markdown, SourceFile, null); return Task.FromResult(document); } @@ -127,14 +127,14 @@ protected override Task GetMinimalParseDocumentAsync(Cancel ct { Title = Rule?.Name; var markdown = GetMarkdown(); - var document = MarkdownParser.MinimalParseStringAsync(markdown, RuleSourceMarkdownPath, null); + var document = MarkdownParser.MinimalParseString(markdown, RuleSourceMarkdownPath, null); return Task.FromResult(document); } protected override Task GetParseDocumentAsync(Cancel ctx) { var markdown = GetMarkdown(); - var document = MarkdownParser.ParseStringAsync(markdown, RuleSourceMarkdownPath, null); + var document = MarkdownParser.ParseString(markdown, RuleSourceMarkdownPath, null); return Task.FromResult(document); } diff --git a/src/Elastic.Markdown/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs index cdba6bcd0..0a99e74e7 100644 --- a/src/Elastic.Markdown/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -43,7 +43,7 @@ public class HtmlWriter( public string Render(string markdown, IFileInfo? source) { source ??= DocumentationSet.Context.ConfigurationPath; - var parsed = DocumentationSet.MarkdownParser.ParseStringAsync(markdown, source, null); + var parsed = DocumentationSet.MarkdownParser.ParseString(markdown, source, null); return MarkdownFile.CreateHtml(parsed); } diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 5dc9886c4..5029c7984 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -9,6 +9,7 @@ using Elastic.Documentation; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Documentation.Configuration.TableOfContents; using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; @@ -132,6 +133,8 @@ public class DocumentationSet : INavigationLookups, IPositionalNavigation public ConcurrentDictionary NavigationRenderResults { get; } = []; + public DiagramRegistry DiagramRegistry { get; } + public DocumentationSet( BuildContext context, ILoggerFactory logFactory, @@ -143,6 +146,7 @@ public DocumentationSet( Source = ContentSourceMoniker.Create(context.Git.RepositoryName, null); SourceDirectory = context.DocumentationSourceDirectory; OutputDirectory = context.DocumentationOutputDirectory; + DiagramRegistry = new DiagramRegistry(logFactory, context); LinkResolver = linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(logFactory, context.Configuration, Aws3LinkIndexReader.CreateAnonymous())); Configuration = context.Configuration; @@ -154,7 +158,7 @@ public DocumentationSet( CrossLinkResolver = LinkResolver, DocumentationFileLookup = DocumentationFileLookup }; - MarkdownParser = new MarkdownParser(context, resolver); + MarkdownParser = new MarkdownParser(context, resolver, DiagramRegistry); Name = Context.Git != GitCheckoutInformation.Unavailable ? Context.Git.RepositoryName diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index a70f8db94..725943cc6 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -29,10 +29,8 @@ public class DiagramBlock(DirectiveBlockParser parser, ParserContext context) : /// public string? EncodedUrl { get; private set; } - /// - /// The local SVG path relative to the output directory - /// - public string? LocalSvgPath { get; private set; } + /// The local SVG Url + public string? LocalSvgUrl { get; private set; } /// /// Content hash for unique identification and caching @@ -56,8 +54,9 @@ public override void FinalizeAndValidate(ParserContext context) // Generate content hash for caching ContentHash = GenerateContentHash(DiagramType, Content); - // Generate local path for cached SVG - LocalSvgPath = GenerateLocalPath(context); + // Generate the local path and url for cached SVG + var localPath = GenerateLocalPath(context); + LocalSvgUrl = localPath.Replace(Path.DirectorySeparatorChar, '/'); // Generate the encoded URL for Kroki try @@ -70,14 +69,14 @@ public override void FinalizeAndValidate(ParserContext context) return; } - // Register diagram for tracking, cleanup, and batch caching - // Use the markdown file's scope directory for proper relative path resolution - var outputDirectory = context.Build.DocumentationOutputDirectory.FullName; + // only register SVG if we can look up the Markdown if (context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown) { - outputDirectory = currentMarkdown.ScopeDirectory.FullName; + var path = context.Build.ReadFileSystem.FileInfo.New(Path.Combine(currentMarkdown.ScopeDirectory.FullName, localPath)); + context.DiagramRegistry.RegisterDiagramForCaching(path, EncodedUrl); } - context.DiagramRegistry.RegisterDiagramForCaching(LocalSvgPath, EncodedUrl, outputDirectory); + else + this.EmitError($"Can not locate markdown source for {context.MarkdownSourcePath} to register diagram for caching."); } private string? ExtractContent() @@ -109,17 +108,13 @@ private string GenerateContentHash(string diagramType, string content) private string GenerateLocalPath(ParserContext context) { - var markdownFileName = "unknown"; - if (context.MarkdownSourcePath?.Name is not null) - { - markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.Name); - } + var markdownFileName = Path.GetFileNameWithoutExtension(context.MarkdownSourcePath.Name); var filename = $"{markdownFileName}-diagram-{DiagramType}-{ContentHash}.svg"; var localPath = Path.Combine("images", "generated-graphs", filename); // Normalize path separators to forward slashes for web compatibility - return localPath.Replace(Path.DirectorySeparatorChar, '/'); + return localPath; } diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml index 1b3433798..e9a50d539 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramView.cshtml @@ -4,9 +4,9 @@ if (diagram?.EncodedUrl != null) {
- @if (!string.IsNullOrEmpty(diagram.LocalSvgPath)) + @if (!string.IsNullOrEmpty(diagram.LocalSvgUrl)) { - @diagram.DiagramType diagram diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs index 4c8a6e2d6..4a04c6f86 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs @@ -294,8 +294,16 @@ private static void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block) var snippet = block.Build.ReadFileSystem.FileInfo.New(block.IncludePath); var parentPath = block.Context.MarkdownParentPath ?? block.Context.MarkdownSourcePath; - var document = MarkdownParser.ParseSnippetAsync(block.Build, block.Context, snippet, parentPath, block.Context.YamlFrontMatter, default) - .GetAwaiter().GetResult(); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = snippet, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + ParentMarkdownPath = parentPath, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseSnippetAsync(snippet, state, Cancel.None).GetAwaiter().GetResult(); var html = document.ToHtml(MarkdownParser.Pipeline); _ = renderer.Write(html); @@ -330,7 +338,15 @@ private static void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock bloc SettingsCollection = settings, RenderMarkdown = s => { - var document = MarkdownParser.ParseMarkdownStringAsync(block.Build, block.Context, s, block.IncludeFrom, block.Context.YamlFrontMatter, MarkdownParser.Pipeline); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = block.IncludeFrom, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseMarkdownString(s, MarkdownParser.Pipeline, state); var html = document.ToHtml(MarkdownParser.Pipeline); return html; } diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs index f5e01bd21..7272591c7 100644 --- a/src/Elastic.Markdown/Myst/MarkdownParser.cs +++ b/src/Elastic.Markdown/Myst/MarkdownParser.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using Cysharp.IO; using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Diagram; using Elastic.Markdown.Helpers; using Elastic.Markdown.Myst.CodeBlocks; using Elastic.Markdown.Myst.Comments; @@ -24,7 +25,7 @@ namespace Elastic.Markdown.Myst; -public partial class MarkdownParser(BuildContext build, IParserResolvers resolvers) +public partial class MarkdownParser(BuildContext build, IParserResolvers resolvers, DiagramRegistry diagramRegistry) { private BuildContext Build { get; } = build; public IParserResolvers Resolvers { get; } = resolvers; @@ -45,30 +46,37 @@ private Task ParseFromFile( YamlFrontMatter = matter, DocumentationFileLookup = Resolvers.DocumentationFileLookup, CrossLinkResolver = Resolvers.CrossLinkResolver, - SkipValidation = skip + SkipValidation = skip, + DiagramRegistry = diagramRegistry }; var context = new ParserContext(state); return ParseAsync(path, context, pipeline, ctx); } - public MarkdownDocument ParseStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter) => - ParseMarkdownStringAsync(markdown, path, matter, Pipeline); + public MarkdownDocument ParseString(string markdown, IFileInfo path, YamlFrontMatter? matter) => + ParseMarkdownString(markdown, path, matter, Pipeline); - public MarkdownDocument MinimalParseStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter) => - ParseMarkdownStringAsync(markdown, path, matter, MinimalPipeline); + public MarkdownDocument MinimalParseString(string markdown, IFileInfo path, YamlFrontMatter? matter) => + ParseMarkdownString(markdown, path, matter, MinimalPipeline); - private MarkdownDocument ParseMarkdownStringAsync(string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) => - ParseMarkdownStringAsync(Build, Resolvers, markdown, path, matter, pipeline); + private MarkdownDocument ParseMarkdownString(string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) => + ParseMarkdownString(Build, Resolvers, markdown, path, matter, pipeline); - public static MarkdownDocument ParseMarkdownStringAsync(BuildContext build, IParserResolvers resolvers, string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) + public MarkdownDocument ParseMarkdownString(BuildContext build, IParserResolvers resolvers, string markdown, IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline) { var state = new ParserState(build) { MarkdownSourcePath = path, YamlFrontMatter = matter, DocumentationFileLookup = resolvers.DocumentationFileLookup, - CrossLinkResolver = resolvers.CrossLinkResolver + CrossLinkResolver = resolvers.CrossLinkResolver, + DiagramRegistry = diagramRegistry }; + return ParseMarkdownString(markdown, pipeline, state); + } + + public static MarkdownDocument ParseMarkdownString(string markdown, MarkdownPipeline pipeline, ParserState state) + { var context = new ParserContext(state); // Preprocess substitutions in link patterns before Markdig parsing @@ -78,27 +86,13 @@ public static MarkdownDocument ParseMarkdownStringAsync(BuildContext build, IPar return markdownDocument; } - public static Task ParseSnippetAsync(BuildContext build, IParserResolvers resolvers, IFileInfo path, IFileInfo parentPath, - YamlFrontMatter? matter, Cancel ctx) + public static Task ParseSnippetAsync(IFileInfo path, ParserState state, Cancel ctx) { - var state = new ParserState(build) - { - MarkdownSourcePath = path, - YamlFrontMatter = matter, - DocumentationFileLookup = resolvers.DocumentationFileLookup, - CrossLinkResolver = resolvers.CrossLinkResolver, - ParentMarkdownPath = parentPath - }; var context = new ParserContext(state); return ParseAsync(path, context, Pipeline, ctx); } - - private static async Task ParseAsync( - IFileInfo path, - MarkdownParserContext context, - MarkdownPipeline pipeline, - Cancel ctx) + private static async Task ParseAsync(IFileInfo path, MarkdownParserContext context, MarkdownPipeline pipeline, Cancel ctx) { string inputMarkdown; if (path.FileSystem is FileSystem) diff --git a/src/Elastic.Markdown/Myst/ParserContext.cs b/src/Elastic.Markdown/Myst/ParserContext.cs index 0ab2e4046..7ad12c159 100644 --- a/src/Elastic.Markdown/Myst/ParserContext.cs +++ b/src/Elastic.Markdown/Myst/ParserContext.cs @@ -45,6 +45,7 @@ public record ParserState(BuildContext Build) : ParserResolvers public required IFileInfo MarkdownSourcePath { get; init; } public required YamlFrontMatter? YamlFrontMatter { get; init; } + public required DiagramRegistry DiagramRegistry { get; init; } public IFileInfo? ParentMarkdownPath { get; init; } public bool SkipValidation { get; init; } @@ -59,7 +60,7 @@ public class ParserContext : MarkdownParserContext, IParserResolvers public string CurrentUrlPath { get; } public YamlFrontMatter? YamlFrontMatter { get; } public BuildContext Build { get; } - public DiagramRegistry DiagramRegistry => Build.DiagramRegistry; + public DiagramRegistry DiagramRegistry { get; } public bool SkipValidation { get; } public Func DocumentationFileLookup { get; } public IReadOnlyDictionary Substitutions { get; } @@ -76,6 +77,7 @@ public ParserContext(ParserState state) CrossLinkResolver = state.CrossLinkResolver; MarkdownSourcePath = state.MarkdownSourcePath; DocumentationFileLookup = state.DocumentationFileLookup; + DiagramRegistry = state.DiagramRegistry; CurrentUrlPath = DocumentationFileLookup(state.ParentMarkdownPath ?? MarkdownSourcePath) is MarkdownFile md ? md.Url diff --git a/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs b/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs index 8fdebf0e6..b7215f45b 100644 --- a/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs +++ b/src/Elastic.Markdown/Myst/Renderers/LlmMarkdown/LlmBlockRenderers.cs @@ -465,8 +465,16 @@ private void WriteIncludeBlock(LlmMarkdownRenderer renderer, IncludeBlock block) try { var parentPath = block.Context.MarkdownParentPath ?? block.Context.MarkdownSourcePath; - var document = MarkdownParser.ParseSnippetAsync(block.Build, block.Context, snippet, parentPath, block.Context.YamlFrontMatter, Cancel.None) - .GetAwaiter().GetResult(); + var state = new ParserState(block.Build) + { + MarkdownSourcePath = snippet, + YamlFrontMatter = block.Context.YamlFrontMatter, + DocumentationFileLookup = block.Context.DocumentationFileLookup, + CrossLinkResolver = block.Context.CrossLinkResolver, + ParentMarkdownPath = parentPath, + DiagramRegistry = block.Context.DiagramRegistry + }; + var document = MarkdownParser.ParseSnippetAsync(snippet, state, Cancel.None).GetAwaiter().GetResult(); _ = renderer.Render(document); } catch (Exception ex) diff --git a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs index 734f6a5a4..020af6abc 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DiagramTests.cs @@ -3,7 +3,11 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions.TestingHelpers; +using Elastic.Documentation; +using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Diagram; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; using Elastic.Markdown.Myst.Directives.Diagram; using FluentAssertions; @@ -38,13 +42,13 @@ flowchart LR public void GeneratesContentHash() => Block!.ContentHash.Should().NotBeNullOrEmpty(); [Fact] - public void GeneratesLocalSvgPath() => Block!.LocalSvgPath.Should().Contain("images/generated-graphs/"); + public void GeneratesLocalSvgUrl() => Block!.LocalSvgUrl.Should().Contain("images/generated-graphs/"); [Fact] - public void LocalSvgPathContainsHash() => Block!.LocalSvgPath.Should().Contain(Block!.ContentHash!); + public void LocalSvgPathContainsHash() => Block!.LocalSvgUrl.Should().Contain(Block!.ContentHash!); [Fact] - public void LocalSvgPathContainsDiagramType() => Block!.LocalSvgPath.Should().Contain("-diagram-mermaid-"); + public void LocalSvgPathContainsDiagramType() => Block!.LocalSvgUrl.Should().Contain("-diagram-mermaid-"); [Fact] public void RendersLocalPathWithFallback() => Html.Should().Contain("onerror=\"this.src='https://kroki.io/mermaid/svg/"); @@ -102,65 +106,69 @@ public void EmptyContentGeneratesError() => public class DiagramRegistryTests { - [Fact] - public void CleanupUnusedDiagramsWithNoActiveFilesCleansAllFiles() - { - var fileSystem = new MockFileSystem(); - var registry = new DiagramRegistry(fileSystem); - registry.RegisterDiagramForCaching("test-path.svg", "http://example.com/test", "/test"); + private MockFileSystem FileSystem { get; } - // Clear registry to simulate no active diagrams - registry.Clear(); + private BuildContext Context { get; } - var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/test")); + private DiagramRegistry Registry { get; } - cleanedCount.Should().Be(0); // No files to clean since directory doesn't exist + public DiagramRegistryTests(ITestOutputHelper output) + { + var collector = new DiagnosticsCollector([]); + var versionsConfig = new VersionsConfiguration + { + VersioningSystems = new Dictionary() + }; + FileSystem = new MockFileSystem(new Dictionary + { + { "docs/index.md", new MockFileData($"# {nameof(DiagramRegistryTests)}") } + }, new MockFileSystemOptions + { + CurrentDirectory = Paths.WorkingDirectoryRoot.FullName + }); + var root = FileSystem.DirectoryInfo.New(Path.Combine(Paths.WorkingDirectoryRoot.FullName, "docs/")); + FileSystem.GenerateDocSetYaml(root); + Context = new BuildContext(collector, FileSystem, versionsConfig); + Registry = new DiagramRegistry(new TestLoggerFactory(output), Context); } [Fact] public void CleanupUnusedDiagramsWithActiveAndUnusedFilesCleansOnlyUnused() { - var fileSystem = new MockFileSystem(); - var registry = new DiagramRegistry(fileSystem); - registry.Clear(); - registry.RegisterDiagramForCaching("images/generated-graphs/active-diagram.svg", "http://example.com/active", "/output"); + var localOutput = FileSystem.DirectoryInfo.New(Path.Combine(Context.DocumentationOutputDirectory.FullName, "output")); + var file = FileSystem.FileInfo.New(Path.Combine(localOutput.FullName, "images", "generated-graphs", "active-diagram.svg")); + Registry.RegisterDiagramForCaching(file, "http://example.com/active"); - fileSystem.AddDirectory("/output/images/generated-graphs"); - fileSystem.AddFile("/output/images/generated-graphs/active-diagram.svg", "active content"); - fileSystem.AddFile("/output/images/generated-graphs/unused-diagram.svg", "unused content"); + FileSystem.AddDirectory(Path.Combine(localOutput.FullName, "images/generated-graphs")); + FileSystem.AddFile(Path.Combine(localOutput.FullName, "images/generated-graphs/active-diagram.svg"), "active content"); + FileSystem.AddFile(Path.Combine(localOutput.FullName, "images/generated-graphs/unused-diagram.svg"), "unused content"); - var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/output")); + var cleanedCount = Registry.CleanupUnusedDiagrams(); cleanedCount.Should().Be(1); - fileSystem.File.Exists("/output/images/generated-graphs/active-diagram.svg").Should().BeTrue(); - fileSystem.File.Exists("/output/images/generated-graphs/unused-diagram.svg").Should().BeFalse(); + FileSystem.File.Exists(Path.Combine(localOutput.FullName, "images/generated-graphs/active-diagram.svg")).Should().BeTrue(); + FileSystem.File.Exists(Path.Combine(localOutput.FullName, "images/generated-graphs/unused-diagram.svg")).Should().BeFalse(); } [Fact] public void CleanupUnusedDiagramsWithNonexistentDirectoryReturnsZero() { - var fileSystem = new MockFileSystem(); - var registry = new DiagramRegistry(fileSystem); - registry.Clear(); - - var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/nonexistent")); - + var cleanedCount = Registry.CleanupUnusedDiagrams(); cleanedCount.Should().Be(0); } [Fact] public void CleanupUnusedDiagramsRemovesEmptyDirectories() { - var fileSystem = new MockFileSystem(); - var registry = new DiagramRegistry(fileSystem); - registry.Clear(); + var localOutput = FileSystem.DirectoryInfo.New(Path.Combine(Context.DocumentationOutputDirectory.FullName, "output")); + var file = FileSystem.FileInfo.New(Path.Combine(localOutput.FullName, "images", "generated-graphs", "unused.svg")); - fileSystem.AddDirectory("/output/images/generated-graphs/subdir"); - fileSystem.AddFile("/output/images/generated-graphs/subdir/unused.svg", "content"); + FileSystem.AddDirectory(file.Directory!.FullName); + FileSystem.AddFile(file.FullName, "content"); - var cleanedCount = registry.CleanupUnusedDiagrams(fileSystem.DirectoryInfo.New("/output")); + var cleanedCount = Registry.CleanupUnusedDiagrams(); cleanedCount.Should().Be(1); - fileSystem.Directory.Exists("/output/images/generated-graphs/subdir").Should().BeFalse(); + FileSystem.Directory.Exists(file.Directory.FullName).Should().BeFalse(); } } diff --git a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj index 5530dffce..189720f32 100644 --- a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj +++ b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj @@ -15,6 +15,7 @@ + From b4c88de914e01b4660e8944d81138dfddc1b3670 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 24 Jul 2025 14:48:10 +0200 Subject: [PATCH 10/11] Fail the build on CI if new cachable SVG files are discovered --- .../Diagram/DiagramRegistry.cs | 8 ++++++++ .../Diagnostics/IDiagnosticsCollector.cs | 2 -- src/Elastic.Markdown/DocumentationGenerator.cs | 2 -- src/tooling/docs-builder/Http/DocumentationWebHost.cs | 7 +------ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs index e971d4c7f..8242568b5 100644 --- a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs +++ b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.IO.Abstractions; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Extensions; using Microsoft.Extensions.Logging; @@ -70,6 +71,13 @@ public async Task CreateDiagramCachedFiles(Cancel ctx) if (_readFileSystem.File.Exists(diagramInfo.OutputFile.FullName)) return; + // If we are running on CI, and we are creating cached files we should fail the build and alert the user to create them + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("CI"))) + { + context.Collector.EmitGlobalError($"Discovered new diagram SVG '{localPath}' please run `docs-builder --force` to ensure a cached version is generated"); + return; + } + // Create the directory if needed var directory = _writeFileSystem.Path.GetDirectoryName(diagramInfo.OutputFile.FullName); if (directory != null && !_writeFileSystem.Directory.Exists(directory)) diff --git a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs index fb1d24f06..86cb66b9e 100644 --- a/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs +++ b/src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs @@ -53,5 +53,3 @@ public static void EmitGlobalWarning(this IDiagnosticsCollector collector, strin public static void EmitGlobalHint(this IDiagnosticsCollector collector, string message) => collector.EmitHint(string.Empty, message); } - - diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index ec6355c1f..a15a85d88 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -161,9 +161,7 @@ public async Task CreateDiagramCachedFiles(Cancel ctx) { var downloadedCount = await DocumentationSet.DiagramRegistry.CreateDiagramCachedFiles(ctx); if (downloadedCount > 0) - { _logger.LogInformation("Downloaded {DownloadedCount} diagram files from Kroki", downloadedCount); - } } /// diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index 5e5ee7e70..52aacacf4 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -193,15 +193,12 @@ private static async Task ServeDocumentationFile(ReloadableGeneratorSta var generator = holder.Generator; const string navPartialSuffix = ".nav.html"; - // Check if the original request is asking for LLM-rendered markdown + // Check if the original request is asking for LLM-rendered Markdown var requestLlmMarkdown = slug.EndsWith(".md"); - var originalSlug = slug; // If requesting .md output, remove the .md extension to find the source file if (requestLlmMarkdown) - { slug = slug[..^3]; // Remove ".md" extension - } if (slug.EndsWith(navPartialSuffix)) { @@ -258,9 +255,7 @@ private static async Task ServeDocumentationFile(ReloadableGeneratorSta { var svgPath = Path.Combine(generator.DocumentationSet.OutputDirectory.FullName, slug.TrimStart('/')); if (File.Exists(svgPath)) - { return Results.File(svgPath, "image/svg+xml"); - } } if (!generator.DocumentationSet.FlatMappedFiles.TryGetValue("404.md", out var notFoundDocumentationFile)) From d84ace321d09ec4f0c2a839f3b20cdfde525e9d5 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 24 Jul 2025 15:01:17 +0200 Subject: [PATCH 11/11] register output file in output folder, we have to write it to both source and output folder though --- .../Diagram/DiagramRegistry.cs | 1 + .../Myst/Directives/Diagram/DiagramBlock.cs | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs index 8242568b5..9751083ff 100644 --- a/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs +++ b/src/Elastic.Documentation.Configuration/Diagram/DiagramRegistry.cs @@ -83,6 +83,7 @@ public async Task CreateDiagramCachedFiles(Cancel ctx) if (directory != null && !_writeFileSystem.Directory.Exists(directory)) _ = _writeFileSystem.Directory.CreateDirectory(directory); + _logger.LogWarning("Creating local diagram: {LocalPath}", localPath); // Download SVG content var svgContent = await _httpClient.GetStringAsync(diagramInfo.EncodedUrl, ct); diff --git a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs index 725943cc6..6feb6e7d1 100644 --- a/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/Diagram/DiagramBlock.cs @@ -72,8 +72,11 @@ public override void FinalizeAndValidate(ParserContext context) // only register SVG if we can look up the Markdown if (context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown) { - var path = context.Build.ReadFileSystem.FileInfo.New(Path.Combine(currentMarkdown.ScopeDirectory.FullName, localPath)); - context.DiagramRegistry.RegisterDiagramForCaching(path, EncodedUrl); + var fs = context.Build.ReadFileSystem; + var scopePath = fs.FileInfo.New(Path.Combine(currentMarkdown.ScopeDirectory.FullName, localPath)); + var relativeScopePath = fs.Path.GetRelativePath(context.Build.DocumentationSourceDirectory.FullName, scopePath.FullName); + var outputPath = fs.FileInfo.New(Path.Combine(context.Build.DocumentationOutputDirectory.FullName, relativeScopePath)); + context.DiagramRegistry.RegisterDiagramForCaching(outputPath, EncodedUrl); } else this.EmitError($"Can not locate markdown source for {context.MarkdownSourcePath} to register diagram for caching.");