In this tutorial, we implement OmniParseBench, Datalab’s open benchmark for document parsers, which grades a parser’s output with 16,288 small pass-or-fail tests across 2,937 pages, from scanned forms and handwriting to arXiv math and dense tables. Rather than trust the leaderboard, we rebuild it from the verdicts Datalab released for 27 parsers, after feeding the scorer a hand-made page to see which mistakes flip a verdict. We then measure ranking certainty, slice it by document type, test whether the two models the card names as helping build tests behave differently on tests a model corrected, and run Tesseract and Indra OCR ourselves on a 50-page sample. Along the way, we find that the top two parsers are statistically tied, the leader changes on most slices, and one launch figure doesn’t match the released files.
Copy CodeCopiedUse a different Browser
import os import re import sys import json import time import html import shutil import platform import traceback import subprocess import collections import urllib.request from pathlib import Path
RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) CURRENT = ["1"]
def banner(title): print("\n" + "=" * 78) print(title) print("=" * 78)
def section(name): def wrap(fn): def run(*a, kw): CURRENT[0] = name.split(".")[0] banner(name) t0 = time.time() try: out = fn(*a, kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).name}: {e}" print(f"\n[!] {name} did not complete: {type(e).name}: {e}") traceback.print_exc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") return run return wrap
def claim(text, held): CLAIMS.append((CURRENT[0], text, bool(held))) return held
banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")
SYSTEM = platform.system()
OPB_VERSION = "1.0.0rc1" # a pre-release: a bare pip install omni-parse-bench resolves to 0.2.0, whose API differs
REPO = "datalab-to/omni_parse_bench"
REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a" # the dataset with the published verdicts of 27 parsers
INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",
"pypdfium2==5.14.0", "huggingface_hub==2.2.0"] # the pins of the package's own Indra server image
WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)
if SYSTEM == "Linux" and shutil.which("tesseract") is None:
scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "
"hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
subprocess.run(["apt-get", "-qq", "update"], check=True)
subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
+ [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)
import numpy as np import pandas as pd import importlib.metadata as md from huggingface_hub import snapshot_download from omni_parse_bench import benchmark, data, harness, markdown, vocab from omni_parse_bench.scorers import unit_tests
print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, " f"{os.cpu_count()} CPU cores") if SYSTEM == "Linux": print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0]) tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True) print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0]) TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0]) print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())}) print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}") RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = ( f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")
We install omni-parse-bench pinned to 1.0.0rc1, because a bare pip install still resolves to 0.2.0, an older release with a different API. The benchmark extra brings the parser adapters, the Hugging Face client, and pyarrow for the manifest, and on Colab we add Tesseract with its script models through apt. We also pin the dataset to one commit, since the repository changes as parsers are added. Printing the families shows the structure the benchmark rests on: present, order, and repeat tests form the text family; table_cell forms the tables family; and layout_kind forms the layout family.
Copy CodeCopiedUse a different Browser
@section("2. How the scorer decides: one hand-built page, five test types, one change at a time") def scorer_anatomy(): T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored", "tags": tags, "args": args, "settings": st or {}} exact = lambda s: {"text": s, "max_diffs": 0} gold = [ T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}), T("total", "present", "html", ["eng"], {"content": exact("Total due")}), T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}), T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}), T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}), T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}), T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."), "after": exact("Conclusion: the plan holds.")}), T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}), T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}), T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}), T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"), "left": exact("Widget")}), T("block_title", "layout_kind", "blocks", ["heading"], {"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}), T("block_total", "layout_kind", "blocks", ["eng"], {"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}), ] gold_file = WORK / "fixture" / "gold" / "demo-p1.json" # a gold file is named after its page gold_file.parent.mkdir(parents=True, exist_ok=True) gold_file.write_text(json.dumps({"tests": gold}, indent=1)) page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf", gt_path=gold_file, outputs=("blocks", "html")) tests = unit_tests.gold(page)
table_a = ("
20232024
Income1,0001,240
" "
Costs600700
") table_b = "
ProductPrice
Widget9.50
" md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |" parts = {"title": "
Quarterly Report
", "total": "
Total due: 1,240.00 USD.
", "growth": "
Growth was -4.8% year on year.
", "ratio": r"
Ratio: 1+ holds.
", "half": r"
Half: 12x in the model.
", "intro": "
Introduction: this report covers the second quarter.
", "revenue": "
Revenue is shown in thousands. Revenue grew in the second quarter.
", "table_a": table_a, "conclusion": "
Conclusion: the plan holds.
", "table_b": table_b} blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly Report"}, {"label": "text", "bbox": [0.10, 0.12, 0.90, 0.16], "text": "Total due: 1,240.00 USD."}, {"label": "table", "bbox": [0.10, 0.40, 0.90, 0.60], "text": "Income 1,000 1,240 Costs 600 700"}]
def pred(change=None, order=None, new_blocks=None): p = dict(parts, **(change or {})) return {"html": "\n".join(p[k] for k in (order or list(parts))), "blocks": blocks if new_blocks is None else new_blocks}
base = {v.id: v.result for v in unit_tests.score(pred(), tests)} print(f"base prediction: {sum(r == 'pass' for r in base.values())} of {len(tests)} tests pass\n") swapped = ["title", "total", "growth", "ratio", "half", "conclusion", "intro", "revenue", "table_a", "table_b"] variants = [ ("flip", "a typo: 'Total dve'", pred({"total": "
Total dve: 1,240.00 USD.
"}), ()), ("flip", "the minus sign dropped: 4.8%", pred({"growth": "
Growth was 4.8% year on year.
"}), ()), ("flip", "another symbol: alpha_2", pred({"ratio": r"
Ratio: 2+ holds.
"}), ()), ("flip", "conclusion before introduction", pred(order=swapped), ()), ("flip", "'Revenues' for the second 'Revenue'", pred({"revenue": "
Revenue is shown in thousands. Revenues grew in the second quarter.
"}), ()), ("flip", "2023 and 2024 headings swapped", pred({"table_a": table_a.replace("20232024", "20242023")}), ()), ("flip", "row label 'Margin' for 'Income'", pred({"table_a": table_a.replace("Income", "Margin")}), ()), ("flip", "a markdown table left in the html", pred({"table_b": md_table_b}), ()), ("flip", "the title's block labelled text", pred(new_blocks=[dict(blocks[0], label="text")] + blocks[1:]), ()), ("control", r"\dfrac12\rightarrow for \frac{1}{2}\to", pred({"half": r"
Half: $\dfrac12\rightarrow x$ in the model.
"}), ()), ("control", "'TOTAL DUE' and '1240.00'", pred({"total": "
TOTAL DUE: 1240.00 USD.
"}), ()), ("control", "markdown table through markdown.to_html", pred({"table_b": markdown.to_html(md_table_b)}), ()), ("unsupported", "no blocks, declared unsupported", {"html": pred()["html"]}, ("blocks",)), ] as_expected = 0 for kind, name, p, unsupported in variants: changed = [v for v in unit_tests.score(p, tests, unsupported) if v.result != base[v.id]] if kind == "flip": ok = len(changed) == 1 and changed[0].result == "fail" elif kind == "control": ok = not changed else: ok = bool(changed) and all(v.result == "unsupported" for v in changed) as_expected += ok shown = "; ".join(f"{v.id} -> {v.result}" + (f" ({v.why})" if v.result == "fail" else "") for v in changed) or "no verdict changes" print(f"{kind:11s} {name:42s} {shown}") s = unit_tests.summarize(unit_tests.lines(unit_tests.score(variants[0][2], tests), tests, page)) print(f"\nsummary of the typo variant: headline {s['headline']:.4f} = mean of " + ", ".join(f"{k} {v['score']:.3f} ({v['tests']} tests)" for k, v in s["families"].items())) claim("every one-change variant flips exactly one verdict to fail, and no control flips any", as_expected == len(variants)) return f"{as_expected} of {len(variants)} hand-built variants behave as expected (9 flips, 3 controls, 1 unsupported)"
scorer_anatomy()
Before trusting a score, we want to see what a verdict means, so we write a gold file by hand with thirteen tests on an invented page and grade predictions with unit_tests.score, the function the benchmark itself uses. A base prediction passes everything, and each variant changes one thing. A one-letter typo, a lost minus sign, a different symbol in an equation, swapped headings and a mislabelled block each flip exactly one test to fail, with a reason attached. The controls still pass: capitals, a dropped thousands separator and dfrac in place of frac, because text is folded and equations are compared after KaTeX renders them.
Copy CodeCopiedUse a different Browser
@section("3. The dataset: 16,288 tests, what they check, and how each answer was established") def dataset_step(): global DATA, PAGES, TESTS, CARD_TEXT snapshot_download(REPO, repo_type="dataset", revision=REV, local_dir=str(WORK / "dataset"), allow_patterns=["manifest.parquet", "gold/*", "results.parquet", "README.md"]) DATA, PAGES = data.read_manifest(str(WORK / "dataset")) rows = [] for p in PAGES: for t
[truncated for AI cost control]