diff --git a/mypy/build.py b/mypy/build.py
index 250a09f5c46c6..17730afc4afce 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -734,7 +734,7 @@ class BuildManager:
                        Semantic analyzer, pass 2
       all_types:       Map {Expression: Type} from all modules (enabled by export_types)
       options:         Build options
-      missing_modules: Set of modules that could not be imported encountered so far
+      missing_modules: Modules that could not be imported (or intentionally skipped)
       stale_modules:   Set of modules that needed to be rechecked (only used by tests)
       fg_deps_meta:    Metadata for fine-grained dependencies caches associated with modules
       fg_deps:         A fine-grained dependency map
@@ -795,7 +795,7 @@ def __init__(
         self.version_id = version_id
         self.modules: dict[str, MypyFile] = {}
         self.import_map: dict[str, set[str]] = {}
-        self.missing_modules: set[str] = set()
+        self.missing_modules: dict[str, int] = {}
         self.fg_deps_meta: dict[str, FgDepMeta] = {}
         # fg_deps holds the dependencies of every module that has been
         # processed. We store this in BuildManager so that we can compute
@@ -2186,9 +2186,17 @@ def write_cache_meta(meta: CacheMeta, manager: BuildManager, meta_file: str) ->
 """
 
 
+class SuppressionReason:
+    NOT_FOUND: Final = 1
+    SKIPPED: Final = 2
+
+
 class ModuleNotFound(Exception):
     """Control flow exception to signal that a module was not found."""
 
+    def __init__(self, reason: int = SuppressionReason.NOT_FOUND) -> None:
+        self.reason = reason
+
 
 class State:
     """The state for a module.
@@ -2319,9 +2327,9 @@ def new_state(
                     root_source,
                     skip_diagnose=temporary,
                 )
-            except ModuleNotFound:
+            except ModuleNotFound as exc:
                 if not temporary:
-                    manager.missing_modules.add(id)
+                    manager.missing_modules[id] = exc.reason
                 raise
             if follow_imports == "silent":
                 ignore_all = True
@@ -3079,9 +3087,13 @@ def suppressed_deps_opts(self) -> bytes:
         buf = WriteBuffer()
         import_options = self.manager.import_options
         for dep in sorted(self.suppressed):
+            # Using .get() is a bit defensive, but just in case we have a bug elsewhere
+            # (e.g. in the daemon), it is better to get a stale cache than a crash.
+            reason = self.manager.missing_modules.get(dep, SuppressionReason.NOT_FOUND)
             if self.priorities.get(dep) != PRI_INDIRECT:
                 write_str_bare(buf, dep)
                 write_bytes_bare(buf, import_options[dep])
+                write_int_bare(buf, reason)
         return buf.getvalue()
 
     def write_cache(self) -> tuple[CacheMeta, str] | None:
@@ -3268,7 +3280,12 @@ def find_module_and_diagnose(
                     skipping_ancestor(manager, id, result, ancestor_for)
                 else:
                     skipping_module(manager, caller_line, caller_state, id, result)
-            raise ModuleNotFound
+            reason = SuppressionReason.SKIPPED
+            if options.ignore_missing_imports:
+                # Performance optimization: when we are ignoring imports, there is no
+                # difference for the caller between skipped import and actually missing one.
+                reason = SuppressionReason.NOT_FOUND
+            raise ModuleNotFound(reason=reason)
         if is_silent_import_module(manager, result) and not root_source:
             follow_imports = "silent"
         return result, follow_imports
@@ -3812,7 +3829,7 @@ def load_graph(
         for dep in st.ancestors + dependencies + st.suppressed:
             ignored = dep in st.suppressed_set and dep not in entry_points
             if ignored and dep not in added:
-                manager.missing_modules.add(dep)
+                manager.missing_modules[dep] = SuppressionReason.NOT_FOUND
                 # TODO: for now we skip this in the daemon as a performance optimization.
                 # This however creates a correctness issue, see #7777 and State.is_fresh().
                 if not manager.use_fine_grained_cache():
@@ -3876,10 +3893,10 @@ def load_graph(
     # modules that are back in graph. We need to do this after the loop to cover an edge
     # case where a namespace package ancestor is shared by a typed and an untyped package.
     for st in graph.values():
-        for dep in st.suppressed:
+        for dep in st.suppressed.copy():
             if dep in graph:
                 st.add_dependency(dep)
-                manager.missing_modules.discard(dep)
+                manager.missing_modules.pop(dep, None)
     # Second, in the initial loop we skip indirect dependencies, so to make indirect dependencies
     # behave more consistently with regular ones, we suppress them manually here (when needed).
     for st in graph.values():
diff --git a/mypy/semanal.py b/mypy/semanal.py
index f38a71cb16e30..1b0b10bfc7277 100644
--- a/mypy/semanal.py
+++ b/mypy/semanal.py
@@ -448,7 +448,7 @@ class SemanticAnalyzer(
     def __init__(
         self,
         modules: dict[str, MypyFile],
-        missing_modules: set[str],
+        missing_modules: dict[str, int],
         incomplete_namespaces: set[str],
         errors: Errors,
         plugin: Plugin,
diff --git a/mypy/server/update.py b/mypy/server/update.py
index 8ecfc95c373da..9f793b73a08c4 100644
--- a/mypy/server/update.py
+++ b/mypy/server/update.py
@@ -128,6 +128,7 @@
     BuildResult,
     Graph,
     State,
+    SuppressionReason,
     load_graph,
     process_fresh_modules,
 )
@@ -595,7 +596,7 @@ def update_module_isolated(
     sources = get_sources(manager.fscache, previous_modules, [(module, path)], followed)
 
     if module in manager.missing_modules:
-        manager.missing_modules.remove(module)
+        del manager.missing_modules[module]
 
     orig_module = module
     orig_state = graph.get(module)
@@ -731,7 +732,8 @@ def delete_module(module_id: str, path: str, graph: Graph, manager: BuildManager
     # If the module is removed from the build but still exists, then
     # we mark it as missing so that it will get picked up by import from still.
     if manager.fscache.isfile(path):
-        manager.missing_modules.add(module_id)
+        # TODO: check if there is an equivalent of #20800 for the daemon.
+        manager.missing_modules[module_id] = SuppressionReason.NOT_FOUND
 
 
 def dedupe_modules(modules: list[tuple[str, str]]) -> list[tuple[str, str]]:
diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test
index 4deb005063265..5d6c188289da0 100644
--- a/test-data/unit/check-incremental.test
+++ b/test-data/unit/check-incremental.test
@@ -7993,3 +7993,22 @@ from a import b  # type: ignore[attr-defined]
 [out]
 main:2: error: Unused "type: ignore" comment
 [out2]
+
+[case testAddedMissingModuleSkip]
+# flags: --follow-imports=skip
+import mod
+[file mod.py.2]
+[out]
+main:2: error: Cannot find implementation or library stub for module named "mod"
+main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+[out2]
+
+[case testDeletedModuleSkip]
+# flags: --follow-imports=skip
+import mod
+[file mod.py]
+[delete mod.py.2]
+[out]
+[out2]
+main:2: error: Cannot find implementation or library stub for module named "mod"
+main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports