Add CLI flag to enable DocC adding minimal per-page HTML content to each index.html file

Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertAction.swift

@@ -30,6 +30,7 @@ public struct ConvertAction: AsyncAction {
     let diagnosticEngine: DiagnosticEngine
 
     private let transformForStaticHosting: Bool
+    private let includeContentInEachHTMLFile: Bool
     private let hostingBasePath: String?
 
     let sourceRepository: SourceRepository?
@@ -64,6 +65,7 @@ public struct ConvertAction: AsyncAction {
     ///   - experimentalEnableCustomTemplates: `true` if the convert action should enable support for custom "header.html" and "footer.html" template files, otherwise `false`.
     ///   - experimentalModifyCatalogWithGeneratedCuration: `true` if the convert action should write documentation extension files containing markdown representations of DocC's automatic curation into the `documentationBundleURL`, otherwise `false`.
     ///   - transformForStaticHosting: `true` if the convert action should process the build documentation archive so that it supports a static hosting environment, otherwise `false`.
+    ///   - includeContentInEachHTMLFile: `true` if the convert action should process each static hosting HTML file so that it includes documentation content for environments without JavaScript enabled, otherwise `false`.
     ///   - allowArbitraryCatalogDirectories: `true` if the convert action should consider the root location as a documentation bundle if it doesn't discover another bundle, otherwise `false`.
     ///   - hostingBasePath: The base path where the built documentation archive will be hosted at.
     ///   - sourceRepository: The source repository where the documentation's sources are hosted.
@@ -91,6 +93,7 @@ public struct ConvertAction: AsyncAction {
         experimentalEnableCustomTemplates: Bool = false,
         experimentalModifyCatalogWithGeneratedCuration: Bool = false,
         transformForStaticHosting: Bool = false,
+        includeContentInEachHTMLFile: Bool = false,
         allowArbitraryCatalogDirectories: Bool = false,
         hostingBasePath: String? = nil,
         sourceRepository: SourceRepository? = nil,
@@ -105,6 +108,7 @@ public struct ConvertAction: AsyncAction {
         self.temporaryDirectory = temporaryDirectory
         self.documentationCoverageOptions = documentationCoverageOptions
         self.transformForStaticHosting = transformForStaticHosting
+        self.includeContentInEachHTMLFile = includeContentInEachHTMLFile
         self.hostingBasePath = hostingBasePath
         self.sourceRepository = sourceRepository
 
@@ -299,9 +303,17 @@ public struct ConvertAction: AsyncAction {
             context: context,
             indexer: indexer,
             enableCustomTemplates: experimentalEnableCustomTemplates,
-            transformForStaticHostingIndexHTML: transformForStaticHosting ? indexHTML : nil,
+            // Don't transform for static hosting if the `FileWritingHTMLContentConsumer` will create per-page index.html files
+            transformForStaticHostingIndexHTML: transformForStaticHosting && !includeContentInEachHTMLFile ? indexHTML : nil,
             bundleID: inputs.id
         )
+
+        let htmlConsumer: FileWritingHTMLContentConsumer?
+        if includeContentInEachHTMLFile, let indexHTML {
+            htmlConsumer = try FileWritingHTMLContentConsumer(targetFolder: temporaryFolder, fileManager: fileManager, htmlTemplate: indexHTML)
+        } else {
+            htmlConsumer = nil
+        }
 
         if experimentalModifyCatalogWithGeneratedCuration, let catalogURL = rootURL {
             let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: catalogURL)
@@ -320,7 +332,7 @@ public struct ConvertAction: AsyncAction {
                 try ConvertActionConverter.convert(
                     context: context,
                     outputConsumer: outputConsumer,
-                    htmlContentConsumer: nil,
+                    htmlContentConsumer: htmlConsumer,
                     sourceRepository: sourceRepository,
                     emitDigest: emitDigest,
                     documentationCoverageOptions: documentationCoverageOptions

Sources/SwiftDocCUtilities/Action/Actions/Convert/FileWritingHTMLContentConsumer.swift

@@ -31,23 +31,42 @@ struct FileWritingHTMLContentConsumer: HTMLContentConsumer {
         var descriptionReplacementRange: Range<String.Index>
 
         init(data: Data) throws {
-            let content = String(decoding: data, as: UTF8.self)
+            var content = String(decoding: data, as: UTF8.self)
 
-            // ???: Should we parse the content with XMLParser instead? If so, what do we do if it's not valid XHTML?
-            let noScriptStart = content.utf8.firstRange(of:  "<noscript>".utf8)!.upperBound
-            let noScriptEnd   = content.utf8.firstRange(of: "</noscript>".utf8)!.lowerBound
-
-            let titleStart = content.utf8.firstRange(of:  "<title>".utf8)!.upperBound
-            let titleEnd   = content.utf8.firstRange(of: "</title>".utf8)!.lowerBound
+            // Ensure that the index.html file has at least a `<head>` and a `<body>`.
+            guard var beforeEndOfHead  = content.utf8.firstRange(of: "</head>".utf8)?.lowerBound,
+                  var afterStartOfBody = content.range(of: "<body[^>]*>", options: .regularExpression)?.upperBound
+            else {
+                struct MissingRequiredTagsError: DescribedError {
+                    let errorDescription = "Missing required `<head>` and `<body>` elements in \"index.html\" file."
+                }
+                throw MissingRequiredTagsError()
+            }
 
-            let beforeHeadEnd = content.utf8.firstRange(of: "</head>".utf8)!.lowerBound
+            if let titleStart = content.utf8.firstRange(of:  "<title>".utf8)?.upperBound,
+               let titleEnd   = content.utf8.firstRange(of: "</title>".utf8)?.lowerBound
+            {
+                titleReplacementRange = titleStart ..< titleEnd
+            } else {
+                content.insert(contentsOf: "<title></title>", at: beforeEndOfHead)
+                content.utf8.formIndex(&beforeEndOfHead,  offsetBy: "<title></title>".utf8.count)
+                content.utf8.formIndex(&afterStartOfBody, offsetBy: "<title></title>".utf8.count)
+                let titleInside = content.utf8.index(beforeEndOfHead, offsetBy: -"</title>".utf8.count)
+                titleReplacementRange = titleInside ..< titleInside
+            }
 
+            if let noScriptStart = content.utf8.firstRange(of:  "<noscript>".utf8)?.upperBound,
+               let noScriptEnd   = content.utf8.firstRange(of: "</noscript>".utf8)?.lowerBound
+            {
+                contentReplacementRange = noScriptStart ..< noScriptEnd
+            } else {
+                content.insert(contentsOf: "<noscript></noscript>", at: afterStartOfBody)
+                let noScriptInside = content.utf8.index(afterStartOfBody, offsetBy: "<noscript>".utf8.count)
+                contentReplacementRange = noScriptInside ..< noScriptInside
+            }
+
             original = content
-            // TODO: If the template doesn't already contain a <noscript> element, add one to the start of the <body> element
-            // TODO: If the template doesn't already contain a <title> element, add one to the end of the <head> element
-            contentReplacementRange     = noScriptStart ..< noScriptEnd
-            titleReplacementRange       = titleStart    ..< titleEnd
-            descriptionReplacementRange = beforeHeadEnd ..< beforeHeadEnd
+            descriptionReplacementRange = beforeEndOfHead ..< beforeEndOfHead
 
             assert(titleReplacementRange.upperBound       < descriptionReplacementRange.lowerBound, "The title replacement range should be before the description replacement range")
             assert(descriptionReplacementRange.upperBound < contentReplacementRange.lowerBound,     "The description replacement range should be before the content replacement range")

Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/ConvertAction+CommandInitialization.swift

@@ -78,6 +78,7 @@ extension ConvertAction {
             experimentalEnableCustomTemplates: convert.experimentalEnableCustomTemplates,
             experimentalModifyCatalogWithGeneratedCuration: convert.experimentalModifyCatalogWithGeneratedCuration,
             transformForStaticHosting: convert.transformForStaticHosting,
+            includeContentInEachHTMLFile: convert.experimentalTransformForStaticHostingWithContent,
             allowArbitraryCatalogDirectories: convert.allowArbitraryCatalogDirectories,
             hostingBasePath: convert.hostingBasePath,
             sourceRepository: SourceRepository(from: convert.sourceRepositoryArguments),

Sources/SwiftDocCUtilities/ArgumentParsing/Subcommands/Convert.swift

@@ -184,6 +184,20 @@ extension Docc {
                 help: "Produce a DocC archive that supports static hosting environments."
             )
             var transformForStaticHosting = true
+
+            @Flag(help: "Include documentation content in each HTML file for static hosting environments.")
+            var experimentalTransformForStaticHostingWithContent = false
+
+            mutating func validate() throws {
+                if experimentalTransformForStaticHostingWithContent, !transformForStaticHosting {
+                    warnAboutDiagnostic(.init(
+                        severity: .warning,
+                        identifier: "org.swift.docc.IgnoredNoTransformForStaticHosting",
+                        summary: "Passing '--experimental-transform-for-static-hosting-with-content' also implies '--transform-for-static-hosting'. Passing '--no-transform-for-static-hosting' has no effect."
+                    ))
+                    transformForStaticHosting = true
+                }
+            }
         }
 
         /// A Boolean value that is true if the DocC archive produced by this conversion will support static hosting environments.
@@ -194,6 +208,12 @@ extension Docc {
             set { hostingOptions.transformForStaticHosting = newValue }
         }
 
+        /// A Boolean value that is true if the DocC archive produced by this conversion will support browsing without JavaScript enabled.
+        public var experimentalTransformForStaticHostingWithContent: Bool {
+            get { hostingOptions.experimentalTransformForStaticHostingWithContent }
+            set { hostingOptions.experimentalTransformForStaticHostingWithContent = newValue }
+        }
+
         /// A user-provided relative path to be used in the archived output
         var hostingBasePath: String? {
             hostingOptions.hostingBasePath

Tests/SwiftDocCUtilitiesTests/ArgumentParsing/ConvertSubcommandTests.swift

@@ -596,6 +596,37 @@ class ConvertSubcommandTests: XCTestCase {
         let disabledFlagConvert = try Docc.Convert.parse(["--disable-mentioned-in"])
         XCTAssertEqual(disabledFlagConvert.enableMentionedIn, false)
     }
+
+    func testStaticHostingWithContentFlag() throws {
+        // The feature is enabled when no flag is passed.
+        let noFlagConvert = try Docc.Convert.parse([])
+        XCTAssertEqual(noFlagConvert.experimentalTransformForStaticHostingWithContent, false)
+
+        let enabledFlagConvert = try Docc.Convert.parse(["--experimental-transform-for-static-hosting-with-content"])
+        XCTAssertEqual(enabledFlagConvert.experimentalTransformForStaticHostingWithContent, true)
+
+        // The '...-transform...-with-content' flag also implies the base '--transform-...' flag.
+        do {
+            let originalErrorLogHandle = Docc.Convert._errorLogHandle
+            let originalDiagnosticFormattingOptions = Docc.Convert._diagnosticFormattingOptions
+            defer {
+                Docc.Convert._errorLogHandle = originalErrorLogHandle
+                Docc.Convert._diagnosticFormattingOptions = originalDiagnosticFormattingOptions
+            }
+
+            let logStorage = LogHandle.LogStorage()
+            Docc.Convert._errorLogHandle = .memory(logStorage)
+            Docc.Convert._diagnosticFormattingOptions = .formatConsoleOutputForTools
+
+            let conflictingFlagsConvert = try Docc.Convert.parse(["--experimental-transform-for-static-hosting-with-content", "--no-transform-for-static-hosting"])
+            XCTAssertEqual(conflictingFlagsConvert.experimentalTransformForStaticHostingWithContent, true)
+            XCTAssertEqual(conflictingFlagsConvert.transformForStaticHosting, true)
+
+            XCTAssertEqual(logStorage.text.trimmingCharacters(in: .whitespacesAndNewlines), """
+            warning: Passing '--experimental-transform-for-static-hosting-with-content' also implies '--transform-for-static-hosting'. Passing '--no-transform-for-static-hosting' has no effect.
+            """)
+        }
+    }
 
     // This test calls ``ConvertOptions.infoPlistFallbacks._unusedVersionForBackwardsCompatibility`` which is deprecated.
     // Deprecating the test silences the deprecation warning when running the tests. It doesn't skip the test.

Tests/SwiftDocCUtilitiesTests/FileWritingHTMLContentConsumerTests.swift

@@ -474,6 +474,108 @@ final class FileWritingHTMLContentConsumerTests: XCTestCase {
         """)
     }
 
+    func testAddsTagsToTemplateIfMissing() async throws {
+        let catalog = Folder(name: "Something.docc", content: [
+            TextFile(name: "RootArticle.md", utf8Content: """
+            # A single article
+
+            This is an _formatted_ article that becomes the root page (because there's only one page).
+            """)
+        ])
+
+        for withTitleTag in [true, false] {
+            for withNoScriptTag in [true, false] {
+                let maybeTitleTag = withTitleTag ? "<title>Documentation</title>" : ""
+                let maybeNoScriptTag = withNoScriptTag ? """
+                  <noscript>
+                    <p>Some existing information inside the no script tag</p>
+                  </noscript>
+                """ : ""
+
+                let htmlTemplate = TextFile(name: "index.html", utf8Content: """
+                <html>
+                  <head>
+                    <meta charset="utf-8" />
+                    <link rel="icon" href="/favicon.ico" />
+                    \(maybeTitleTag)
+                  </head>
+                  <body>
+                  \(maybeNoScriptTag)
+                    <div id="app"></div>
+                  </body>
+                </html>
+                """)
+
+                let fileSystem = try TestFileSystem(folders: [
+                    Folder(name: "path", content: [
+                        Folder(name: "to", content: [
+                            catalog
+                        ])
+                    ]),
+                    Folder(name: "template", content: [
+                        htmlTemplate
+                    ]),
+                    Folder(name: "output-dir", content: [])
+                ])
+
+                let (inputs, dataProvider) = try DocumentationContext.InputsProvider(fileManager: fileSystem)
+                    .inputsAndDataProvider(startingPoint: URL(fileURLWithPath: "/path/to/\(catalog.name)"), options: .init())
+
+                let context = try await DocumentationContext(bundle: inputs, dataProvider: dataProvider, configuration: .init())
+
+                let htmlConsumer = try FileWritingHTMLContentConsumer(
+                    targetFolder: URL(fileURLWithPath: "/output-dir"),
+                    fileManager: fileSystem,
+                    htmlTemplate: URL(fileURLWithPath: "/template/index.html"),
+                    prettyPrintOutput: true
+                )
+
+                _ = try ConvertActionConverter.convert(
+                    context: context,
+                    outputConsumer: TestOutputConsumer(),
+                    htmlContentConsumer: htmlConsumer,
+                    sourceRepository: nil,
+                    emitDigest: false,
+                    documentationCoverageOptions: .noCoverage
+                )
+
+                // Because the TestOutputConsumer below, doesn't create any files, we only expect the HTML files in the output directory
+                XCTAssertEqual(fileSystem.dump(subHierarchyFrom: "/output-dir"), """
+                output-dir/
+                ╰─ documentation/
+                   ╰─ rootarticle/
+                      ╰─ index.html
+                """)
+
+                try assert(readHTML: fileSystem.contents(of: URL(fileURLWithPath: "/output-dir/documentation/rootarticle/index.html")), matches: """
+                <html>
+                  <head>
+                    <meta charset="utf-8" />
+                    <link rel="icon" href="/favicon.ico" />
+                    <title>A single article</title>
+                    <meta content="This is an formatted article that becomes the root page (because there’s only one page)." name="description"/>
+                  </head>
+                  <body>
+                    <noscript>
+                      <article>
+                        <section>
+                          <ul>
+                            <li>RootArticle</li>
+                          </ul>
+                          <p>
+                          Article</p>
+                          <h1>RootArticle</h1>
+                          <p>This is an <i> formatted</i> article that becomes the root page (because there’s only one page).</p>
+                        </section>
+                      </article>
+                    </noscript>
+                    <div id="app"></div>
+                  </body>
+                </html>
+                """)
+            }
+        }
+    }
 }
 
 // MARK: Helpers

features.json

@@ -14,6 +14,9 @@
     },
     {
       "name": "synthesized-landing-page-name"
+    },
+    {
+      "name": "static-hosting-with-content"
     }
   ]
 }