Search: respect spacing from block elements when indexing (#11658)

HTML tags can be divided in two categories: inline and block elements.
Inline elements do not start on a new line, while block elements start on a new line.
This gives block elements an implicit spacing that is not present in
inline elements. If there are two tags next to each other,
and one of them is a block element, there will be a space between them.
Or if the two tags are inline elements, there will be no space between them.

Author: stsewd

readthedocs/search/parsers.py

@@ -15,6 +15,45 @@ class GenericParser:
     # Limit that matches the ``index.mapping.nested_objects.limit`` ES setting.
     max_inner_documents = 10000
 
+    # Block level elements have an implicit line break before and after them.
+    # List taken from: https://www.w3schools.com/htmL/html_blocks.asp.
+    block_level_elements = [
+        "address",
+        "article",
+        "aside",
+        "blockquote",
+        "canvas",
+        "dd",
+        "div",
+        "dl",
+        "dt",
+        "fieldset",
+        "figcaption",
+        "figure",
+        "footer",
+        "form",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "header",
+        "hr",
+        "li",
+        "main",
+        "nav",
+        "noscript",
+        "ol",
+        "p",
+        "pre",
+        "section",
+        "table",
+        "tfoot",
+        "ul",
+        "video",
+    ]
+
     def __init__(self, version):
         self.version = version
         self.project = self.version.project
@@ -334,7 +373,12 @@ def _parse_section_content(self, tag, *, depth=0):
                 )
 
             if content:
-                contents.append(content)
+                is_block_level_element = next_tag.tag in self.block_level_elements
+                if is_block_level_element:
+                    # Add a line break before and after a block level element.
+                    contents.append(f"\n{content}\n")
+                else:
+                    contents.append(content)
             next_tag = next_tag.next
 
         return self._parse_content("".join(contents)), section_found

readthedocs/search/tests/data/sphinx/in/page.html

@@ -166,6 +166,16 @@ <h2>Footnotes and domains<a class="headerlink" href="#footnotes-and-domains" tit
   </div>
   <!-- End of footnote -->
 
+  <!-- Definition list -->
+  <section id="development">
+    <h2>Development<a class="headerlink" href="#development" title="Permalink to this heading"></a></h2>
+    <dl class="simple">
+      <!-- NOTE: leave this as a single line to test a bug related to spacing -->
+      <dt><a class="reference internal" href="contributing.html"><span class="doc">Contributing</span></a></dt><dd><p>How to contribute changes to the theme.</p></dd>
+    </dl>
+  </section>
+  <!-- End of definition list -->
+
 </div>
     </main>
   </body>

readthedocs/search/tests/data/sphinx/out/page.json

@@ -52,6 +52,11 @@
       "title": "Footnotes and domains",
       "content": ""
     },
+    {
+      "content": "Contributing How to contribute changes to the theme.",
+      "id": "development",
+      "title": "Development"
+    },
     {
       "id": "subsub-title",
       "title": "Subsub title",