Fix tests, add way to disable ocr

Author: VikParuchuri

marker/builders/line.py

@@ -36,10 +36,6 @@ class LineBuilder(BaseBuilder):
         "The batch size to use for the ocr error detection model.",
         "Default is None, which will use the default batch size for the model.",
     ] = None
-    enable_table_ocr: Annotated[
-        bool,
-        "Whether to skip OCR on tables.  The TableProcessor will re-OCR them.  Only enable if the TableProcessor is not running.",
-    ] = False
     layout_coverage_min_lines: Annotated[
         int,
         "The minimum number of PdfProvider lines that must be covered by the layout model",
@@ -54,17 +50,10 @@ class LineBuilder(BaseBuilder):
         float,
         "If less pages than this threshold are good, OCR will happen in the document.  Otherwise it will not.",
     ] = 0.85
-    provider_line_detected_line_min_overlap_pct: Annotated[
-        float,
-        "The percentage of a provider line that has to be covered by a detected line",
-    ] = 0.1
     provider_line_provider_line_min_overlap_pct: Annotated[
         float,
         "The percentage of a provider line that has to be covered by a detected line",
-    ] = 0.1
-    line_vertical_merge_threshold: Annotated[
-        int, "The maximum pixel distance between y1s for two lines to be merged"
-    ] = 8
+    ] = 0.15
     excluded_for_coverage: Annotated[
         Tuple[BlockTypes],
         "A list of block types to exclude from the layout coverage check.",
@@ -86,6 +75,10 @@ class LineBuilder(BaseBuilder):
         bool,
         "Disable tqdm progress bars.",
     ] = False
+    disable_ocr: Annotated[
+        bool,
+        "Disable OCR for the document. This will only use the lines from the provider.",
+    ] = False
     keep_chars: Annotated[bool, "Keep individual characters."] = False
 
     def __init__(
@@ -169,6 +162,9 @@ def get_all_lines(self, document: Document, provider: PdfProvider):
                     ),  # Ensure provider lines don't overflow the page or intersect
                 ]
             )
+            if self.disable_ocr:
+                provider_lines_good = True
+
             layout_good.append(provider_lines_good)
 
         run_detection = [not good for good in layout_good]
@@ -191,12 +187,12 @@ def get_all_lines(self, document: Document, provider: PdfProvider):
             )
 
             # Setup detection results
+            detection_boxes = []
             if detection_result:
                 detection_boxes = [
                     PolygonBox(polygon=box.polygon) for box in detection_result.bboxes
                 ]
-            else:
-                detection_boxes = []
+
             detection_boxes = sort_text_lines(detection_boxes)
 
             if provider_lines_good:
@@ -257,6 +253,7 @@ def check_line_overlaps(
         provider_bboxes = [line.line.polygon.bbox for line in provider_lines]
         # Add a small margin to account for minor overflows
         page_bbox = document_page.polygon.expand(5, 5).bbox
+
         for bbox in provider_bboxes:
             if bbox[0] < page_bbox[0]:
                 return False
@@ -275,7 +272,7 @@ def check_line_overlaps(
             )
 
             # There should be one intersection with itself
-            if intersect_counts > 1:
+            if intersect_counts > 2:
                 return False
 
         return True

tests/builders/test_document_builder.py

@@ -4,20 +4,40 @@
 from marker.schema.text.line import Line
 
 
+@pytest.mark.filename("thinkpython.pdf")
 @pytest.mark.config({"page_range": [0]})
 def test_document_builder(pdf_document):
     first_page = pdf_document.pages[0]
-    assert first_page.structure[0] == '/page/0/SectionHeader/0'
+    assert first_page.structure[0] == "/page/0/SectionHeader/0"
 
     first_block = first_page.get_block(first_page.structure[0])
     assert first_block.block_type == BlockTypes.SectionHeader
-    assert first_block.text_extraction_method == 'pdftext'
+    assert first_block.text_extraction_method == "pdftext"
 
     first_text_block: Line = first_page.get_block(first_block.structure[0])
     assert first_text_block.block_type == BlockTypes.Line
 
     first_span = first_page.get_block(first_text_block.structure[0])
     assert first_span.block_type == BlockTypes.Span
-    assert first_span.text == 'Subspace Adversarial Training'
-    assert first_span.font == 'NimbusRomNo9L-Medi'
-    assert first_span.formats == ['plain']
+    assert first_span.text == "Think Python"
+    assert first_span.font == "URWPalladioL-Roma"
+    assert first_span.formats == ["plain"]
+
+
+@pytest.mark.config({"page_range": [0]})
+def test_document_builder_inline_eq(pdf_document):
+    first_page = pdf_document.pages[0]
+    assert first_page.structure[0] == "/page/0/SectionHeader/0"
+
+    first_block = first_page.get_block(first_page.structure[0])
+    assert first_block.block_type == BlockTypes.SectionHeader
+    assert first_block.text_extraction_method == "surya"
+
+    first_text_block: Line = first_page.get_block(first_block.structure[0])
+    assert first_text_block.block_type == BlockTypes.Line
+
+    first_span = first_page.get_block(first_text_block.structure[0])
+    assert first_span.block_type == BlockTypes.Span
+    assert first_span.text == "Subspace Adversarial Training"
+    assert first_span.font == "NimbusRomNo9L-Medi"
+    assert first_span.formats == ["plain"]

tests/builders/test_layout_replace.py

@@ -8,8 +8,11 @@
 from marker.schema.registry import get_block_class
 
 
+@pytest.mark.filename("thinkpython.pdf")
 @pytest.mark.config({"page_range": [0]})
-def test_layout_replace(request, config, doc_provider, layout_model, ocr_error_model, detection_model):
+def test_layout_replace(
+    request, config, doc_provider, layout_model, ocr_error_model, detection_model
+):
     # The llm layout builder replaces blocks - this makes sure text is still merged properly
     layout_builder = LayoutBuilder(layout_model, config)
     line_builder = LineBuilder(detection_model, ocr_error_model, config)
@@ -35,8 +38,4 @@ def test_layout_replace(request, config, doc_provider, layout_model, ocr_error_m
     renderer = MarkdownRenderer(config)
     rendered = renderer(document)
 
-    assert "worst-case perturbations" in rendered.markdown
-    assert "projected gradient descent" in rendered.markdown
-
-
-
+    assert "Think Python" in rendered.markdown

tests/converters/test_ocr_converter.py

@@ -35,7 +35,7 @@ def check_bboxes(page: OCRJSONPageOutput, lines):
 
 @pytest.mark.config({"page_range": [0]})
 def test_ocr_converter(config, model_dict, temp_doc):
-    _ocr_converter(config, model_dict, temp_doc, 83, 2)
+    _ocr_converter(config, model_dict, temp_doc, 85, 2)
 
 
 @pytest.mark.filename("pres.pdf")

tests/converters/test_pdf_converter.py

@@ -6,7 +6,7 @@
 
 
 @pytest.mark.output_format("markdown")
-@pytest.mark.config({"page_range": [0, 1, 2, 3, 7]})
+@pytest.mark.config({"page_range": [0, 1, 2, 3, 7], "disable_ocr": True})
 def test_pdf_converter(pdf_converter: PdfConverter, temp_doc):
     markdown_output: MarkdownOutput = pdf_converter(temp_doc.name)
     markdown = markdown_output.markdown
@@ -79,7 +79,7 @@ def test_pptx_converter(pdf_converter: PdfConverter, temp_doc):
 
 
 @pytest.mark.output_format("markdown")
-@pytest.mark.config({"page_range": [0, 1, 2, 3, 7]})
+@pytest.mark.config({"page_range": [0, 1, 2, 3, 7], "disable_ocr": True})
 def test_pdf_converter_bytes(pdf_converter: PdfConverter, temp_doc):
     with open(temp_doc.name, "rb") as f:
         data = f.read()

tests/renderers/test_markdown_renderer.py

@@ -5,13 +5,22 @@
 from marker.schema.blocks import TableCell
 
 
-@pytest.mark.config({"page_range": [0]})
+@pytest.mark.config({"page_range": [0], "disable_ocr": True})
 def test_markdown_renderer(pdf_document):
     renderer = MarkdownRenderer()
     md = renderer(pdf_document).markdown
 
     # Verify markdown
-    assert '# Subspace Adversarial Training' in md
+    assert "# Subspace Adversarial Training" in md
+
+
+@pytest.mark.config({"page_range": [0]})
+def test_markdown_renderer_auto_ocr(pdf_document):
+    renderer = MarkdownRenderer()
+    md = renderer(pdf_document).markdown
+
+    # Verify markdown
+    assert "Subspace Adversarial Training" in md
 
 
 @pytest.mark.config({"page_range": [0, 1], "paginate_output": True})
@@ -29,12 +38,14 @@ def test_markdown_renderer_pagination_blank_last_page(pdf_document):
     last_page = pdf_document.pages[-1]
     last_page.children = []
     last_page.structure = []
-    
+
     renderer = MarkdownRenderer({"paginate_output": True})
     md = renderer(pdf_document).markdown
-    
+
     # Should end with pagination marker and preserve trailing newlines
-    assert md.endswith("}\n\n") or md.endswith("}------------------------------------------------\n\n")
+    assert md.endswith("}\n\n") or md.endswith(
+        "}------------------------------------------------\n\n"
+    )
 
 
 @pytest.mark.config({"page_range": [0, 1]})
@@ -48,9 +59,10 @@ def test_markdown_renderer_metadata(pdf_document):
 def test_markdown_renderer_images(pdf_document):
     renderer = MarkdownRenderer({"extract_images": False})
     markdown_output = renderer(pdf_document)
-    
+
     assert len(markdown_output.images) == 0
-    assert '![](' not in markdown_output.markdown
+    assert "![](" not in markdown_output.markdown
+
 
 @pytest.mark.config({"page_range": [5]})
 def test_markdown_renderer_tables(pdf_document):
@@ -74,5 +86,3 @@ def test_markdown_renderer_tables(pdf_document):
     renderer = MarkdownRenderer()
     md = renderer(pdf_document).markdown
     assert "54 <i>.45</i> 67<br>89 $x$" in md
-
-