From 53740cb9e9dbfd8481aa93448acd3224a4e01e70 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 13:49:12 +0000 Subject: [PATCH 1/2] perf(converter): lazy-import markitdown to cut startup and enable slim packaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markitdown is only used for the non-PDF, non-Markdown ingest branch, but it was imported at module top level — so every `import openkb` eagerly pulled in markitdown → magika → onnxruntime (tens of MB, slow). Move the import into the branch that actually uses it. Effects (measured): - `import openkb.cli` cold start 4.64s → 3.53s (~24% faster); magika / onnxruntime / markitdown no longer load unless an Office-format document is converted. - A packaged (PyInstaller) build can now exclude magika/onnxruntime for a slimmer artifact when Office-format ingest isn't needed. Previously the unconditional module-level markitdown import made magika load-bearing at startup, so excluding it crashed the whole app on launch. markitdown appends warning filters on import but leaves the CLI's front "ignore" filter at index 0, so no re-suppression is needed at the new import site. test_converter patches `markitdown.MarkItDown` (the real attribute the local import binds) instead of the removed `openkb.converter.MarkItDown`. Claude-Session: https://claude.ai/code/session_01KZyUSGAzVL9yxpsWWPv6Y2 --- openkb/converter.py | 11 +++++++++-- tests/test_converter.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/openkb/converter.py b/openkb/converter.py index 710d7a77..2cf72abf 100644 --- a/openkb/converter.py +++ b/openkb/converter.py @@ -11,7 +11,6 @@ from pathlib import Path import pymupdf -from markitdown import MarkItDown from openkb.config import load_config from openkb.images import convert_pdf_with_images, copy_relative_images, extract_base64_images @@ -230,7 +229,15 @@ def convert_document( # Use pymupdf dict-mode for PDFs: text + images inline at correct positions markdown = convert_pdf_with_images(src, doc_name, images_dir) else: - # Non-PDF, non-MD: use markitdown (docx, pptx, html, etc.) + # Non-PDF, non-MD: use markitdown (docx, pptx, html, etc.). + # Imported lazily: markitdown pulls in magika → onnxruntime (tens + # of MB, slow to import) that only this branch needs. Keeping it + # out of module import makes `import openkb` cheap and lets a + # packaged build drop those deps when Office-format ingest isn't + # required. markitdown appends warning filters on import but leaves + # the CLI's front "ignore" filter in place, so no re-suppress here. + from markitdown import MarkItDown + mid = MarkItDown() result = mid.convert(str(src), keep_data_uris=True) markdown = result.text_content diff --git a/tests/test_converter.py b/tests/test_converter.py index 1349b6fb..1c265a47 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -145,7 +145,7 @@ def test_docx_conversion_enables_keep_data_uris(self, kb_dir, tmp_path): mock_result.text_content = "![](data:image/png;base64,abc123)" with ( - patch("openkb.converter.MarkItDown") as mock_markitdown, + patch("markitdown.MarkItDown") as mock_markitdown, patch( "openkb.converter.extract_base64_images", return_value="converted markdown", From fb9a6d070584f0d2d4e848747e76ff353bd29864 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 16:24:13 +0000 Subject: [PATCH 2/2] test(converter): guard that importing converter stays markitdown-free Adds a regression test that spawns a fresh interpreter and asserts `import openkb.converter` loads none of markitdown / magika / onnxruntime. This locks in the lazy-import optimization: if a future change reintroduces a module-level markitdown import, startup cost and the slim-packaging benefit regress silently while every functional test still passes. The check runs in a subprocess so it is deterministic regardless of what earlier tests imported into this process's sys.modules. Claude-Session: https://claude.ai/code/session_01KZyUSGAzVL9yxpsWWPv6Y2 --- tests/test_converter.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_converter.py b/tests/test_converter.py index 1c265a47..64adf1b5 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -2,6 +2,8 @@ from __future__ import annotations +import subprocess +import sys from unittest.mock import MagicMock, patch from openkb.converter import convert_document, get_pdf_page_count @@ -164,6 +166,36 @@ def test_docx_conversion_enables_keep_data_uris(self, kb_dir, tmp_path): assert result.source_path.read_text(encoding="utf-8") == "converted markdown" +# --------------------------------------------------------------------------- +# Lazy markitdown import (guards the deferred-import optimization) +# --------------------------------------------------------------------------- + + +class TestLazyMarkItDownImport: + def test_importing_converter_does_not_load_markitdown(self): + """`import openkb.converter` must not pull in markitdown/magika/onnxruntime. + + markitdown is imported lazily inside convert_document's Office-format + branch so `import openkb` stays cheap and a packaged build can drop + magika/onnxruntime. This runs in a fresh interpreter (a clean + sys.modules) so the assertion is deterministic regardless of what other + tests in this process already imported. + """ + heavy = ("markitdown", "magika", "onnxruntime") + code = ( + "import sys, openkb.converter; " + f"print(','.join(m for m in {heavy!r} if m in sys.modules))" + ) + proc = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + loaded = proc.stdout.strip() + assert loaded == "", f"import openkb.converter eagerly loaded: {loaded}" + + # --------------------------------------------------------------------------- # _registry_path # ---------------------------------------------------------------------------