New semantic analyzer: Fix (de)serialization of broken named tuples

mypy/build.py

@@ -1794,8 +1794,13 @@ def patch_dependency_parents(self) -> None:
  semantic analyzer will perform this patch for us when processing stale
  SCCs.
  """
+ Analyzer = Union[SemanticAnalyzerPass2, NewSemanticAnalyzer] # noqa
+ if self.manager.options.new_semantic_analyzer:
+ analyzer = self.manager.new_semantic_analyzer # type: Analyzer
+ else:
+ analyzer = self.manager.semantic_analyzer
  for dep in self.dependencies:
-self.manager.semantic_analyzer.add_submodules_to_parent_modules(dep, True)
+analyzer.add_submodules_to_parent_modules(dep, True)
 
  def fix_suppressed_dependencies(self, graph: Graph) -> None:
  """Corrects whether dependencies are considered stale in silent mode.

mypy/newsemanal/semanal.py

@@ -3955,6 +3955,28 @@ def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
  module_hidden=module_hidden)
  return self.add_symbol_table_node(name, symbol, context)
 
+ def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None:
+ """Same as above, but skipping the local namespace.
+
+ This doesn't check for previous definition and is only used
+ for serialization of method-level classes.
+
+ Classes defined within methods can be exposed through an
+ attribute type, but method-level symbol tables aren't serialized.
+ This method can be used to add such classes to an enclosing,
+ serialized symbol table.
+ """
+ # TODO: currently this is only used by named tuples. Use this method
+ # also by typed dicts and normal classes, see issue #6422.
+ if self.type is not None:
+ names = self.type.names
+ kind = MDEF
+ else:
+ names = self.globals
+ kind = GDEF
+ symbol = SymbolTableNode(kind, node)
+ names[name] = symbol
+
  def current_symbol_table(self) -> SymbolTable:
  if self.is_func_scope():
  assert self.locals[-1] is not None

mypy/newsemanal/semanal_namedtuple.py

@@ -179,7 +179,18 @@ def check_namedtuple(self,
  return True, info
  name = cast(Union[StrExpr, BytesExpr, UnicodeExpr], call.args[0]).value
  if name != var_name or is_func_scope:
- # Give it a unique name derived from the line number.
+ # There are three special cases where need to give it a unique name derived
+ # from the line number:
+ # * There is a name mismatch with l.h.s., therefore we need to disambiguate
+ # situations like:
+ # A = NamedTuple('Same', [('x', int)])
+ # B = NamedTuple('Same', [('y', str)])
+ # * This is a base class expression, since it often matches the class name:
+ # class NT(NamedTuple('NT', [...])):
+ # ...
+ # * This is a local (function or method level) named tuple, since
+ # two methods of a class can define a named tuple with the same name,
+ # and they will be stored in the same namespace (see below).
  name += '@' + str(call.line)
  if len(defaults) > 0:
  default_items = {
@@ -189,15 +200,31 @@ def check_namedtuple(self,
  else:
  default_items = {}
  info = self.build_namedtuple_typeinfo(name, items, types, default_items)
- # Store it as a global just in case it would remain anonymous.
- # (Or in the nearest class if there is one.)
- self.store_namedtuple_info(info, var_name or name, call, is_typed)
+ # If var_name is not None (i.e. this is not a base class expression), we always
+ # store the generated TypeInfo under var_name in the current scope, so that
+ # other definitions can use it.
+ if var_name:
+ self.store_namedtuple_info(info, var_name, call, is_typed)
+ # There are three cases where we need to store the generated TypeInfo
+ # second time (for the purpose of serialization):
+ # * If there is a name mismatch like One = NamedTuple('Other', [...])
+ # we also store the info under name 'Other@lineno', this is needed
+ # because classes are (de)serialized using their actual fullname, not
+ # the name of l.h.s.
+ # * If this is a method level named tuple. It can leak from the method
+ # via assignment to self attribute and therefore needs to be serialized
+ # (local namespaces are not serialized).
+ # * If it is a base class expression. It was not stored above, since
+ # there is no var_name (but it still needs to be serialized
+ # since it is in MRO of some class).
+ if name != var_name or is_func_scope:
+ # NOTE: we skip local namespaces since they are not serialized.
+ self.api.add_symbol_skip_local(name, info)
  return True, info
 
  def store_namedtuple_info(self, info: TypeInfo, name: str,
  call: CallExpr, is_typed: bool) -> None:
- stnode = SymbolTableNode(GDEF, info)
- self.api.add_symbol_table_node(name, stnode)
+ self.api.add_symbol(name, info, call)
  call.analyzed = NamedTupleExpr(info, is_typed=is_typed)
  call.analyzed.set_line(call.line, call.column)
 

mypy/newsemanal/semanal_shared.py

@@ -5,7 +5,8 @@
 from mypy_extensions import trait
 
 from mypy.nodes import (
- Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF
+ Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF,
+ SymbolNode
 )
 from mypy.util import correct_relative_import
 from mypy.types import Type, FunctionLike, Instance
@@ -122,7 +123,23 @@ def schedule_patch(self, priority: int, fn: Callable[[], None]) -> None:
 
  @abstractmethod
  def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> bool:
- """Add node to global symbol table (or to nearest class if there is one)."""
+ """Add node to the current symbol table."""
+ raise NotImplementedError
+
+ @abstractmethod
+ def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
+ module_public: bool = True, module_hidden: bool = False) -> bool:
+ """Add symbol to the current symbol table."""
+ raise NotImplementedError
+
+ @abstractmethod
+ def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None:
+ """Add symbol to the current symbol table, skipping locals.
+
+ This is used to store symbol nodes in a symbol table that
+ is going to be serialized (local namespaces are not serialized).
+ See implementation docstring for more details.
+ """
  raise NotImplementedError
 
  @abstractmethod

test-data/unit/check-incremental.test

@@ -4931,3 +4931,40 @@ import pack.mod
 import a
 [out]
 [out2]
+
+[case testNewAnalyzerIncrementalBrokenNamedTuple]
+# flags: --new-semantic-analyzer
+import a
+[file a.py]
+from b import NT
+x: NT
+[file a.py.2]
+from b import NT
+x: NT
+reveal_type(x)
+[file b.py]
+from typing import NamedTuple
+NT = NamedTuple('BadName', [('x', int)])
+[out]
+[out2]
+tmp/a.py:3: error: Revealed type is 'Tuple[builtins.int, fallback=b.BadName@2]'
+
+[case testNewAnalyzerIncrementalMethodNamedTuple]
+# flags: --new-semantic-analyzer
+import a
+[file a.py]
+from b import C
+x: C
+[file a.py.2]
+from b import C
+x: C
+reveal_type(x.h)
+[file b.py]
+from typing import NamedTuple
+class C:
+ def __init__(self) -> None:
+ self.h: Hidden
+ Hidden = NamedTuple('Hidden', [('x', int)])
+[out]
+[out2]
+tmp/a.py:3: error: Revealed type is 'Tuple[builtins.int, fallback=b.C.Hidden@5]'