[6.3] Experimental markdown output

Sources/DocCCommandLine/Action/Actions/Convert/ConvertFileWritingConsumer.swift

@@ -11,7 +11,7 @@
 import Foundation
 import SwiftDocC
 
-struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
+struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer, ConvertOutputMarkdownConsumer {
     var targetFolder: URL
     var bundleRootFolder: URL?
     var fileManager: any FileManagerProtocol
@@ -68,6 +68,21 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
         indexer?.index(renderNode)
     }
 
+    func consume(markdownNode: WritableMarkdownOutputNode) throws {
+        try renderNodeWriter.write(markdownNode)
+    }
+
+    func consume(markdownManifest: MarkdownOutputManifest) throws {
+        let url = targetFolder.appendingPathComponent("\(markdownManifest.title)-markdown-manifest.json", isDirectory: false)
+        let encoder = JSONEncoder()
+        encoder.outputFormatting = [.sortedKeys, .withoutEscapingSlashes]
+        #if DEBUG
+        encoder.outputFormatting.insert(.prettyPrinted)
+        #endif
+        let data = try encoder.encode(markdownManifest)
+        try fileManager.createFile(at: url, contents: data)
+    }
+
     func consume(externalRenderNode: ExternalRenderNode) throws {
         // Index the external node, if indexing is enabled.
         indexer?.index(externalRenderNode)

Sources/DocCCommandLine/Action/Actions/Convert/JSONEncodingRenderNodeWriter.swift

@@ -89,6 +89,25 @@ class JSONEncodingRenderNodeWriter {
         }
     }
 
+    /// Writes a markdown node to a file at a location based on the node's relative URL.
+    ///
+    /// If the target path to the markdown file includes intermediate folders that don't exist, the writer object will ask the file manager, with which it was created, to
+    /// create those intermediate folders before writing the markdown file.
+    ///
+    /// - Parameters:
+    ///   - markdownNode: The node which the writer object writes
+    func write(_ markdownNode: WritableMarkdownOutputNode) throws {
+        let fileSafePath = NodeURLGenerator.fileSafeReferencePath(
+            markdownNode.identifier,
+            lowercased: true
+        )
+
+        try write(
+            markdownNode.node.generateDataRepresentation(),
+            toFileSafePath: "data/\(fileSafePath).md"
+        )
+    }
+
     func write(_ data: Data, toFileSafePath fileSafePath: String) throws {
         // The path on disk to write the data at.
         let fileURL = targetFolder.appendingPathComponent(fileSafePath, isDirectory: false)

Sources/DocCCommandLine/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift

@@ -25,6 +25,8 @@ extension ConvertAction {
         FeatureFlags.current.isExperimentalOverloadedSymbolPresentationEnabled = convert.enableExperimentalOverloadedSymbolPresentation
         FeatureFlags.current.isMentionedInEnabled = convert.enableMentionedIn
         FeatureFlags.current.isParametersAndReturnsValidationEnabled = convert.enableParametersAndReturnsValidation
+        FeatureFlags.current.isExperimentalMarkdownOutputEnabled = convert.enableExperimentalMarkdownOutput
+        FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled = convert.enableExperimentalMarkdownOutputManifest
 
         // If the user-provided a URL for an external link resolver, attempt to
         // initialize an `OutOfProcessReferenceResolver` with the provided URL.

Sources/DocCCommandLine/ArgumentParsing/Subcommands/Convert.swift

@@ -543,6 +543,12 @@ extension Docc {
             @available(*, deprecated, message: "This flag is unused and only exist for backwards compatibility")
             var _unusedExperimentalMentionedInFlagForBackwardsCompatibility = false
 
+            @Flag(help: "Experimental: Create markdown versions of documents")
+            var enableExperimentalMarkdownOutput = false
+
+            @Flag(help: "Experimental: Create manifest file of markdown outputs. Ignored if --enable-experimental-markdown-output is not set.")
+            var enableExperimentalMarkdownOutputManifest = false
+
             @Flag(
                 name: .customLong("parameters-and-returns-validation"),
                 inversion: .prefixedEnableDisable,
@@ -637,6 +643,18 @@ extension Docc {
             get { featureFlags.enableExperimentalOverloadedSymbolPresentation }
             set { featureFlags.enableExperimentalOverloadedSymbolPresentation = newValue }
         }
+
+        /// A user-provided value that is true if the user enables experimental markdown output
+        public var enableExperimentalMarkdownOutput: Bool {
+            get { featureFlags.enableExperimentalMarkdownOutput }
+            set { featureFlags.enableExperimentalMarkdownOutput = newValue }
+        }
+
+        /// A user-provided value that is true if the user enables experimental markdown output
+        public var enableExperimentalMarkdownOutputManifest: Bool {
+            get { featureFlags.enableExperimentalMarkdownOutputManifest }
+            set { featureFlags.enableExperimentalMarkdownOutputManifest = newValue }
+        }
 
         /// A user-provided value that is true if the user enables experimental automatically generated "mentioned in"
         /// links on symbols.

Sources/SwiftDocC/CMakeLists.txt

@@ -161,6 +161,8 @@ add_library(SwiftDocC
   Model/DocumentationNode.swift
   Model/Identifier.swift
   Model/Kind.swift
+  Model/MarkdownOutput/Translation/MarkdownOutputSemanticVisitor.swift
+  Model/MarkdownOutput/Translation/MarkdownOutputMarkdownWalker.swift
   Model/Markup+parsing.swift
   Model/Name.swift
   Model/ParametersAndReturnValidator.swift
@@ -400,6 +402,8 @@ add_library(SwiftDocC
   Servers/DocumentationSchemeHandler.swift
   Servers/FileServer.swift
   SourceRepository/SourceRepository.swift
+  SwiftDocCMarkdownOutput/MarkdownOutputManifest.swift
+  SwiftDocCMarkdownOutput/MarkdownOutputNode.swift
   Utility/Checksum.swift
   Utility/Collection+ConcurrentPerform.swift
   Utility/CollectionChanges.swift

Sources/SwiftDocC/Converter/DocumentationContextConverter.swift

@@ -113,4 +113,21 @@ public class DocumentationContextConverter {
         )
         return translator.visit(node.semantic) as? RenderNode
     }
+
+    /// Converts a documentation node to a markdown node.
+    /// - Parameters:
+    ///   - node: The documentation node to convert.
+    /// - Returns: The markdown node representation of the documentation node.
+    internal func markdownOutput(for node: DocumentationNode) -> CollectedMarkdownOutput? {
+        guard !node.isVirtual else {
+            return nil
+        }
+
+        var visitor = MarkdownOutputSemanticVisitor(context: context, node: node)
+
+        if let node = visitor.createOutput() {
+            return CollectedMarkdownOutput(identifier: visitor.identifier, node: node, manifest: visitor.manifest)
+        }
+        return nil
+    }
 }

Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift

@@ -80,6 +80,7 @@ package enum ConvertActionConverter {
         var assets = [RenderReferenceType : [any RenderReference]]()
         var coverageInfo = [CoverageDataEntry]()
         let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure()
+        var markdownManifest = MarkdownOutputManifest(title: context.inputs.displayName, documents: [])
 
         // An inner function to gather problems for errors encountered during the conversion.
         //
@@ -131,6 +132,21 @@ package enum ConvertActionConverter {
                         return
                     }
 
+                    if FeatureFlags.current.isExperimentalMarkdownOutputEnabled,
+                       let markdownConsumer = outputConsumer as? (any ConvertOutputMarkdownConsumer),
+                       let markdownNode = converter.markdownOutput(for: entity)
+                    {
+                        try markdownConsumer.consume(markdownNode: markdownNode.writable)
+                        if FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled,
+                           let manifest = markdownNode.manifest
+                        {
+                            resultsGroup.async(queue: resultsSyncQueue) {
+                                markdownManifest.documents.formUnion(manifest.documents)
+                                markdownManifest.relationships.formUnion(manifest.relationships)
+                            }
+                        }
+                    }
+
                     try outputConsumer.consume(renderNode: renderNode)
 
                     switch documentationCoverageOptions.level {
@@ -235,6 +251,12 @@ package enum ConvertActionConverter {
                 }
             }
         }
+
+        if FeatureFlags.current.isExperimentalMarkdownOutputManifestEnabled,
+           let markdownConsumer = outputConsumer as? (any ConvertOutputMarkdownConsumer)
+        {
+            try markdownConsumer.consume(markdownManifest: markdownManifest)
+        }
 
         switch documentationCoverageOptions.level {
         case .detailed, .brief:

Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift

@@ -48,6 +48,15 @@ public protocol ConvertOutputConsumer {
 
     /// Consumes a file representation of the local link resolution information.
     func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws
+
+}
+
+package protocol ConvertOutputMarkdownConsumer {
+    /// Consumes a markdown output node
+    func consume(markdownNode: WritableMarkdownOutputNode) throws
+
+    /// Consumes a markdown output manifest
+    func consume(markdownManifest: MarkdownOutputManifest) throws
 }
 
 // Default implementations that discard the documentation conversion products, for consumers that don't need these