Merge pull request #821 from datalab-to/dev

Bump surya version

Author: VikParuchuri

README.md

@@ -80,8 +80,8 @@ pip install marker-pdf[full]
 First, some configuration:
 
 - Your torch device will be automatically detected, but you can override this.  For example, `TORCH_DEVICE=cuda`.
-- Some PDFs, even digital ones, have bad text in them.  Set the `format_lines` flag to ensure the bad lines are fixed and formatted. You can also set `--force_ocr` to force OCR on all lines, or the `strip_existing_ocr` to keep all digital text, and strip out any existing OCR text.
-- If you care about inline math, set `format_lines` to automatically convert inline math to LaTeX.
+- Some PDFs, even digital ones, have bad text in them.  Set `--force_ocr` to force OCR on all lines, or the `strip_existing_ocr` to keep all digital text, and strip out any existing OCR text.
+- If you care about inline math, set `force_ocr` to convert inline math to LaTeX.
 
 ## Interactive App
 
@@ -106,8 +106,7 @@ Options:
 - `--output_dir PATH`: Directory where output files will be saved. Defaults to the value specified in settings.OUTPUT_DIR.
 - `--paginate_output`: Paginates the output, using `\n\n{PAGE_NUMBER}` followed by `-` * 48, then `\n\n` 
 - `--use_llm`: Uses an LLM to improve accuracy.  You will need to configure the LLM backend - see [below](#llm-services).
-- `--format_lines`: Reformat all lines using a local OCR model (inline math, underlines, bold, etc.).  This will give very good quality math output.
-- `--force_ocr`: Force OCR processing on the entire document, even for pages that might contain extractable text.
+- `--force_ocr`: Force OCR processing on the entire document, even for pages that might contain extractable text.  This will also format inline math properly.
 - `--block_correction_prompt`: if LLM mode is active, an optional prompt that will be used to correct the output of marker.  This is useful for custom formatting or logic that you want to apply to the output.
 - `--strip_existing_ocr`: Remove all existing OCR text in the document and re-OCR with surya.
 - `--redo_inline_math`: If you want the absolute highest quality inline math conversion, use this along with `--use_llm`.
@@ -232,7 +231,7 @@ marker_single FILENAME --use_llm --force_layout_block Table --converter_cls mark
 
 ### OCR Only
 
-If you only want to run OCR, you can also do that through the `OCRConverter`.  Set `--keep_chars` to keep individual characters and bounding boxes.  You can also set `--force_ocr` and `--format_lines` with this converter.
+If you only want to run OCR, you can also do that through the `OCRConverter`.  Set `--keep_chars` to keep individual characters and bounding boxes.
 
 ```python
 from marker.converters.ocr import OCRConverter
@@ -556,4 +555,4 @@ PDF is a tricky format, so marker will not always work perfectly.  Here are some
 - Very complex layouts, with nested tables and forms, may not work
 - Forms may not be rendered well
 
-Note: Passing the `--use_llm` and `--format_lines` flags will mostly solve these issues.
+Note: Passing the `--use_llm` and `--force_ocr` flags will mostly solve these issues.

benchmarks/throughput/main.py

@@ -25,7 +25,6 @@ def get_next_pdf(ds: datasets.Dataset, i: int):
 
 def single_batch(
     batch_size: int,
-    format_lines: bool,
     num_threads: int,
     force_ocr: bool,
     quantize: bool,
@@ -83,7 +82,6 @@ def single_batch(
                     artifact_dict=model_dict,
                     config={
                         "disable_tqdm": worker_id > 0,
-                        "format_lines": format_lines,
                         "page_range": page_range,
                         "force_ocr": force_ocr,
                     },
@@ -104,14 +102,12 @@ def single_batch(
 @click.command(help="Benchmark PDF to MD conversion throughput.")
 @click.option("--workers", default=1, help="Number of workers to use.")
 @click.option("--batch_size", default=1, help="Batch size for inference.")
-@click.option("--format_lines", is_flag=True, help="Format lines in the output.")
 @click.option("--force_ocr", is_flag=True, help="Force OCR on all pages.")
 @click.option("--quantize", is_flag=True, help="Use quantized model.")
 @click.option("--compile", is_flag=True, help="Use compiled model.")
 def main(
     workers: int,
     batch_size: int,
-    format_lines: bool,
     force_ocr: bool,
     quantize: bool,
     compile: bool,
@@ -127,7 +123,6 @@ def main(
             executor.submit(
                 single_batch,
                 batch_size,
-                format_lines,
                 cpus_per_worker,
                 force_ocr,
                 quantize,