From e0451ced44288a1209f77e7262d187a3d2949531 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Thu, 2 Apr 2026 17:55:48 -0700 Subject: [PATCH 1/4] Use WAL with SQLite cache, fix close This is the more modern way to manage concurrency with SQLite In our case, it means concurrent mypy runs using the cache will wait for each other, rather than fail SQLite also claims this is faster, but I haven't yet done a good profile (If you are profiling this, note that WAL is a persistent setting, so you will want to delete the cache) Finally, I also explicitly close the connection in main. This is relevant to this change, because it forces checkpointing of the WAL, which reduces disk space and means the cache.db remains a single self-contained file in regular use --- mypy/main.py | 14 ++++++++++++-- mypy/metastore.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 14148720269a4..dd782b91efbc3 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -189,6 +189,12 @@ def main( list([res]) # noqa: C410 +class BuildResultThunk: + # We pass this around so that we avoid freeing memory, which is slow + def __init__(self, build_result: build.BuildResult | None) -> None: + self._result = build_result + + def run_build( sources: list[BuildSource], options: Options, @@ -196,7 +202,7 @@ def run_build( t0: float, stdout: TextIO, stderr: TextIO, -) -> tuple[build.BuildResult | None, list[str], bool]: +) -> tuple[BuildResultThunk | None, list[str], bool]: formatter = util.FancyFormatter( stdout, stderr, options.hide_error_codes, hide_success=bool(options.output) ) @@ -227,8 +233,12 @@ def flush_errors(filename: str | None, new_messages: list[str], serious: bool) - blockers = True if not e.use_stdout: serious = True + + if res: + res.manager.metastore.close() + maybe_write_junit_xml(time.time() - t0, serious, messages, messages_by_file, options) - return res, messages, blockers + return BuildResultThunk(res), messages, blockers def show_messages( diff --git a/mypy/metastore.py b/mypy/metastore.py index 64839bf8a79c0..86ed8dec8f6b7 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -163,6 +163,7 @@ def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection: # but without this flag, commits are *very* slow, especially when using HDDs, # see https://www.sqlite.org/faq.html#q19 for details. db.execute("PRAGMA synchronous=OFF") + db.execute("PRAGMA journal_mode=WAL") db.executescript(SCHEMA) return db From d2b41760f26d61d9d68acbf6daf074b54e0779fc Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Thu, 2 Apr 2026 18:17:45 -0700 Subject: [PATCH 2/4] attribute error --- mypy/metastore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/metastore.py b/mypy/metastore.py index 86ed8dec8f6b7..65236545f1920 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -173,8 +173,8 @@ def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None: # We check startswith instead of equality because the version # will have already been appended by the time the cache dir is # passed here. + self.db = None if cache_dir_prefix.startswith(os.devnull): - self.db = None return os.makedirs(cache_dir_prefix, exist_ok=True) From 4613cb826e0930f9af265b0fa1d95c77737cb71a Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Thu, 2 Apr 2026 19:47:24 -0700 Subject: [PATCH 3/4] check and set wal --- mypy/metastore.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypy/metastore.py b/mypy/metastore.py index 65236545f1920..3ce27ac3eaeb7 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -163,7 +163,11 @@ def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection: # but without this flag, commits are *very* slow, especially when using HDDs, # see https://www.sqlite.org/faq.html#q19 for details. db.execute("PRAGMA synchronous=OFF") - db.execute("PRAGMA journal_mode=WAL") + + mode = db.execute("PRAGMA journal_mode").fetchone()[0] + if mode.lower() != "wal": + mode = db.execute("PRAGMA journal_mode=WAL").fetchone()[0] + db.executescript(SCHEMA) return db From 6befc59cce1a2f4e21974e976d0825f5ea9e0ba0 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Thu, 2 Apr 2026 20:09:08 -0700 Subject: [PATCH 4/4] hack --- mypy/metastore.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/metastore.py b/mypy/metastore.py index 3ce27ac3eaeb7..757825027895e 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -166,7 +166,10 @@ def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection: mode = db.execute("PRAGMA journal_mode").fetchone()[0] if mode.lower() != "wal": - mode = db.execute("PRAGMA journal_mode=WAL").fetchone()[0] + try: + db.execute("PRAGMA journal_mode=WAL") + except sqlite3.OperationalError: + pass db.executescript(SCHEMA) return db