From 1ce2f215caf6ede2ee8726d08bec72e57e6cd096 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Sat, 4 Mar 2023 17:38:50 +0000 Subject: [PATCH 01/12] Add test to demonstrate issue #9655 This test currently fails with this error: AssertionError: Command 3 (dmypy check -- bar.py) did not give expected output --- Captured stderr call --- Expected: bar.py:2: error: Unused "type: ignore" comment (diff) == Return code: 1 (diff) Actual: (empty) It demonstrates a bug that when an module is removed using `FineGrainedBuildManager.update` because it is not "seen" by `fine_grained_increment_follow_imports`, then "unused type: ignore" warnings disappear from subsequent checks. Ref: https://github.com/python/mypy/issues/9655 --- test-data/unit/daemon.test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index ca2c969d2f5e..07eaff0c64d4 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -615,3 +615,20 @@ b: str from demo.test import a [file demo/test.py] a: int + +[case testUnusedTypeIgnorePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --warn-unused-ignores --no-error-summary --hide-error-codes +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: Unused "type: ignore" comment +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore From 7d9c06a108d01903df6acca238eef1f0c97cf90c Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Sun, 12 Mar 2023 22:23:51 +0000 Subject: [PATCH 02/12] Prove ignores without error codes also disappear This test fails with the error: AssertionError: Command 3 (dmypy check -- bar.py) did not give expected output --- Captured stderr call --- Expected: bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] (diff) == Return code: 1 (diff) Actual: (empty) This test illustrates that '"type: ignore" comment without error code' errors currently disappear in the same way that 'Unused "type: ignore"' errors do as described in #9655. Ref: https://github.com/python/mypy/issues/9655 --- test-data/unit/daemon.test | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 07eaff0c64d4..4c6f49d63df8 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -632,3 +632,20 @@ bar.py:2: error: Unused "type: ignore" comment [file bar.py] from foo.empty import * a = 1 # type: ignore + +[case testTypeIgnoreWithoutCodePreservedOnRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code ignore-without-code --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:2: error: "type: ignore" comment without error code [ignore-without-code] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +a = 1 # type: ignore From 9cd625da3a69197128d4a411549672ef4266a162 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Sun, 12 Mar 2023 23:25:29 +0000 Subject: [PATCH 03/12] Prove possibly-undefined errors disappear This test fails with the error: AssertionError: Command 3 (dmypy check -- bar.py) did not give expected output --- Captured stderr call --- Expected: bar.py:4: error: Name "a" may be undefined [possibly-undefined] (diff) == Return code: 1 (diff) Actual: (empty) This test illustrates that possibly-undefined errors currently disappear in the same way that 'Unused "type: ignore"' errors do as described in #9655. Ref: https://github.com/python/mypy/issues/9655 --- test-data/unit/daemon.test | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 4c6f49d63df8..15d1f64b49cb 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -649,3 +649,22 @@ bar.py:2: error: "type: ignore" comment without error code [ignore-without-code [file bar.py] from foo.empty import * a = 1 # type: ignore + +[case testPossiblyUndefinedVarsPreservedAfterRerun] +-- Regression test for https://github.com/python/mypy/issues/9655 +$ dmypy start -- --enable-error-code possibly-undefined --no-error-summary +Daemon started +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 +$ dmypy check -- bar.py +bar.py:4: error: Name "a" may be undefined [possibly-undefined] +== Return code: 1 + +[file foo/__init__.py] +[file foo/empty.py] +[file bar.py] +from foo.empty import * +if False: + a = 1 +a From 94e5df4ee6480edf5061369fec6927c1859b7348 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Fri, 31 Mar 2023 23:30:08 +0100 Subject: [PATCH 04/12] Test run-running dmypy after a file is altered These tests show how some errors disappear on a re-run of dmypy after a file is altered. Co-authored-by: David Seddon --- test-data/unit/fine-grained.test | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 165a2089b466..ed3b9d90c1c3 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10505,3 +10505,27 @@ from pkg.sub import modb [out] == + +[case testUnusedTypeIgnorePreservedAfterChange] +# flags: --warn-unused-ignores --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: Unused "type: ignore" comment +== +main.py:1: error: Unused "type: ignore" comment + +[case testTypeIgnoreWithoutCodePreservedAfterChange] +# flags: --enable-error-code ignore-without-code --no-error-summary +[file main.py] +a = 1 # type: ignore +[file main.py.2] +a = 1 # type: ignore +# Comment to trigger reload. +[out] +main.py:1: error: "type: ignore" comment without error code +== +main.py:1: error: "type: ignore" comment without error code From b5cc83f7333a263de52447c7405b1dbc7016a64e Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Thu, 13 Apr 2023 18:30:19 +0100 Subject: [PATCH 05/12] Add failing tests showing disappearing errors This shows two cases where errors disappear from the second run of dmypy. The first shows a way that "unused type ignore" errors can disappear. The case is a little complicated, but I can't yet work out how to make it smaller. The second case shows how "module X has not attribute Y" errors can disappear. --- test-data/unit/daemon.test | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 15d1f64b49cb..3a01b2aacd96 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -668,3 +668,42 @@ from foo.empty import * if False: a = 1 a + +[case testUnusedTypeIgnorePreservedOnRerunWithIgnoredMissingImports] +$ dmypy start -- --no-error-summary --ignore-missing-imports --warn-unused-ignores +Daemon started +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 +$ dmypy check foo +foo/main.py:3: error: Unused "type: ignore" comment [unused-ignore] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/empty.py] +[file foo/__init__.py] +from foo.main import * +from unused.submodule import * +[file foo/main.py] +from foo import empty +from foo.does_not_exist import * +a = 1 # type: ignore + +[case testModuleDoesNotExistPreservedOnRerun] +$ dmypy start -- --no-error-summary --ignore-missing-imports +Daemon started +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 +$ dmypy check foo +foo/main.py:1: error: Module "foo" has no attribute "does_not_exist" [attr-defined] +== Return code: 1 + +[file unused/__init__.py] +[file unused/submodule.py] +[file foo/__init__.py] +from foo.main import * +[file foo/main.py] +from foo import does_not_exist +from unused.submodule import * From 99eda138789d26aeac006cb4df9512252c676fd7 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Thu, 20 Jul 2023 15:24:52 +0100 Subject: [PATCH 06/12] Fix disappearing errors when re-running dmypy check This which fixes issue https://github.com/python/mypy/issues/9655 wherein some types of error would be lost when a file was re-processed by dmypy. This also fixes another error where sometimes files would not be re-processed by dmypy if the only error in the file was either "unused type ignore" or "ignore without code". --- mypy/errors.py | 4 ++++ mypy/server/update.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/mypy/errors.py b/mypy/errors.py index eabe96a2dc73..f67087953887 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -725,6 +725,8 @@ def generate_unused_ignore_errors(self, file: str) -> None: blocker=False, only_once=False, allow_dups=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) @@ -777,6 +779,8 @@ def generate_ignore_without_code_errors( blocker=False, only_once=False, allow_dups=False, + origin=(self.file, [line]), + target=self.target_module, ) self._add_error_info(file, info) diff --git a/mypy/server/update.py b/mypy/server/update.py index 0cc7a2229514..0371d5d40266 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -667,6 +667,8 @@ def restore(ids: list[str]) -> None: state.type_check_first_pass() state.type_check_second_pass() state.detect_possibly_undefined_vars() + state.generate_unused_ignore_notes() + state.generate_ignore_without_code_notes() t2 = time.time() state.finish_passes() t3 = time.time() @@ -1028,6 +1030,10 @@ def key(node: FineGrainedDeferredNode) -> int: if graph[module_id].type_checker().check_second_pass(): more = True + graph[module_id].detect_possibly_undefined_vars() + graph[module_id].generate_unused_ignore_notes() + graph[module_id].generate_ignore_without_code_notes() + if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) From d798c5ab96d3e5c4af8e9bbf7aa8bfa90d311f2f Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Wed, 12 Apr 2023 16:02:45 +0100 Subject: [PATCH 07/12] Add a regression test for type ignores in dmypy This catches a regression caused by the previous attempt to fix #9655 where "type: ignore" comments are erroneously marked as unused in re-runs of dmypy. Ref: https://github.com/python/mypy/pull/14835 --- test-data/unit/daemon.test | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 3a01b2aacd96..874117111a21 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -707,3 +707,24 @@ from foo.main import * [file foo/main.py] from foo import does_not_exist from unused.submodule import * + +[case testReturnTypeIgnoreAfterUnknownImport] +-- Return type ignores after unknown imports and unused modules are respected on the second pass. +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:2: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +from unused.empty import * +import a_module_which_does_not_exist +def is_foo() -> str: + return True # type: ignore From ad9424143391d82e7945d7fd16521ff5de9bfa82 Mon Sep 17 00:00:00 2001 From: David Seddon Date: Tue, 2 Jan 2024 11:19:33 +0000 Subject: [PATCH 08/12] Add regression test for attrs plugin This change shows our branch has a regression in the attrs plugin when re-running dmypy. The error produced when running this test is: Expected: foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports == Return code: 1 Actual: foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports foo.py:8: error: Unused "type: ignore" comment [unused-ignore] (diff) == Return code: 1 --- test-data/unit/daemon.test | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index 874117111a21..3b58fb87c810 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -728,3 +728,27 @@ from unused.empty import * import a_module_which_does_not_exist def is_foo() -> str: return True # type: ignore + +[case testAttrsTypeIgnoreAfterUnknownImport] +$ dmypy start -- --warn-unused-ignores --no-error-summary +Daemon started +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 +$ dmypy check -- foo.py +foo.py:3: error: Cannot find implementation or library stub for module named "a_module_which_does_not_exist" [import-not-found] +foo.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports +== Return code: 1 + +[file unused/__init__.py] +[file unused/empty.py] +[file foo.py] +import attr +from unused.empty import * +import a_module_which_does_not_exist + +@attr.frozen +class A: + def __init__(self) -> None: + self.__attrs_init__() # type: ignore[attr-defined] From 22777be332184bf5e218bdf5a3252ebad291f216 Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Tue, 2 Jan 2024 11:59:02 +0000 Subject: [PATCH 09/12] Don't remove modules with descendants Modules are removed from the fine-grained build if they have not been "seen". This change ensures that we don't remove ancestor modules if their descendants have been seen. --- mypy/dmypy_server.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index b4c3fe8fe0dc..4dc3dd68f749 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -684,10 +684,11 @@ def refresh_file(module: str, path: str) -> list[str]: # Find all original modules in graph that were not reached -- they are deleted. to_delete = [] + seen_and_ancestors = self._seen_and_ancestors(seen) for module_id in orig_modules: if module_id not in graph: continue - if module_id not in seen: + if module_id not in seen_and_ancestors: module_path = graph[module_id].path assert module_path is not None to_delete.append((module_id, module_path)) @@ -715,6 +716,29 @@ def refresh_file(module: str, path: str) -> list[str]: return messages + def _seen_and_ancestors(self, seen: set[str]) -> set[str]: + """Return the set of seen modules along with any ancestors not already in the set. + + For example, given this set: + + {"foo", "foo.bar", "a.b.c"} + + ... we would expect this set to be returned: + + {"foo", "foo.bar", "a.b.c", "a.b", "a"} + + This is used to stop us from deleting ancestor modules from the graph + when their descendants have been seen. + """ + seen_paths = seen.copy() + for module_path in seen: + while module_path := module_path.rpartition(".")[0]: + if module_path in seen_paths: + break + else: + seen_paths.add(module_path) + return seen_paths + def find_reachable_changed_modules( self, roots: list[BuildSource], From 64cbe6b360f45af33e15b418cb739c60db6aa3ae Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Thu, 11 Jan 2024 13:29:04 +0000 Subject: [PATCH 10/12] Remove unrequired changes to reprocess_nodes The problem this solved has been addressed in another way, so we don't need to do this re-analysis any more in order to fix the bug we've been chasing. This change is a partial revert of "Fix disappearing errors when re-running dmypy check" from a few commits back. --- mypy/server/update.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mypy/server/update.py b/mypy/server/update.py index 0371d5d40266..2a0d9546e4d5 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -1030,10 +1030,6 @@ def key(node: FineGrainedDeferredNode) -> int: if graph[module_id].type_checker().check_second_pass(): more = True - graph[module_id].detect_possibly_undefined_vars() - graph[module_id].generate_unused_ignore_notes() - graph[module_id].generate_ignore_without_code_notes() - if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) From d58936a53cc1152df033a9f494cfb0f9b804a99a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 00:11:00 +0000 Subject: [PATCH 11/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test-data/unit/daemon.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/daemon.test b/test-data/unit/daemon.test index ab3d89601b53..295eb4000d81 100644 --- a/test-data/unit/daemon.test +++ b/test-data/unit/daemon.test @@ -801,4 +801,4 @@ Found 1 error in 1 file (checked 1 source file) == Return code: 1 [file test.py] from xml.etree.ElementTree import Element -1 + 'a' \ No newline at end of file +1 + 'a' From 6dd9c2e38af0aca6f41d6b6fed860b07c4d510ad Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 21 Jun 2025 00:07:49 +0100 Subject: [PATCH 12/12] Delete part that is likely not needed anymore --- mypy/dmypy_server.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 24fea2fdb52a..33e9e07477ca 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -693,11 +693,10 @@ def refresh_file(module: str, path: str) -> list[str]: # Find all original modules in graph that were not reached -- they are deleted. to_delete = [] - seen_and_ancestors = self._seen_and_ancestors(seen) for module_id in orig_modules: if module_id not in graph: continue - if module_id not in seen_and_ancestors: + if module_id not in seen: module_path = graph[module_id].path assert module_path is not None to_delete.append((module_id, module_path)) @@ -725,29 +724,6 @@ def refresh_file(module: str, path: str) -> list[str]: return messages - def _seen_and_ancestors(self, seen: set[str]) -> set[str]: - """Return the set of seen modules along with any ancestors not already in the set. - - For example, given this set: - - {"foo", "foo.bar", "a.b.c"} - - ... we would expect this set to be returned: - - {"foo", "foo.bar", "a.b.c", "a.b", "a"} - - This is used to stop us from deleting ancestor modules from the graph - when their descendants have been seen. - """ - seen_paths = seen.copy() - for module_path in seen: - while module_path := module_path.rpartition(".")[0]: - if module_path in seen_paths: - break - else: - seen_paths.add(module_path) - return seen_paths - def find_reachable_changed_modules( self, roots: list[BuildSource],