Use PEP 585 syntax in Python 2, protobuf & _ast stubs, where possible

[AlexWaygood]

There are some differences with Python 3 stubs:

• We have to allow ContextManager to be imported from typing, as there is no contextlib.AbstractContextManager class in Python 2.
• We have to allow OrderedDict to be imported from typing_extensions, as there is no collections.OrderedDict class in Python 2.

[AlexWaygood]

Refs PyCQA/flake8-pyi#97

[AlexWaygood]

I used a similar script to #6717

importastimportreimportsubprocessimportsysfromcollectionsimportdefaultdictfromitertoolsimportchainfrompathlibimportPathfromtypingimportNamedTupleclassDeleteableImport(NamedTuple):
old: strreplacement: strclassNewImport(NamedTuple):
text: strindentation: intIMPORTED_FROM_BUILTINS_NOT_TYPING=frozenset({"List", "FrozenSet", "Set", "Dict", "Tuple", "Type"})
IMPORTED_FROM_TYPING_NOT_TYPING_EXTENSIONS=frozenset(
{
# collections.abc aliases"Awaitable",
"Coroutine",
"AsyncIterable",
"AsyncIterator",
"AsyncGenerator",
# typing aliases"Protocol",
"runtime_checkable",
"ClassVar",
"NewType",
"overload",
"Text",
"NoReturn",
}
)
# The values in the mapping are what these are called in `collections`IMPORTED_FROM_COLLECTIONS_NOT_TYPING_OR_TYPING_EXTENSIONS= {
"Counter": "Counter",
"Deque": "deque",
"DefaultDict": "defaultdict",
"ChainMap": "ChainMap",
}
FAILURES= []
deffix_bad_syntax(path: Path) ->None:
withopen(path) asf:
stub=f.read()
lines=stub.splitlines()
tree=ast.parse(stub)
imports_to_delete= {}
imports_to_add=defaultdict(list)
classes_from_typing=set()
classes_from_typing_extensions=set()
classBadImportFinder(ast.NodeVisitor):
defvisit_ImportFrom(self, node: ast.ImportFrom) ->None:
ifnode.modulenotin {"typing", "typing_extensions"}:
returnbad_builtins_classes_in_this_import=set()
bad_collections_classes_in_this_import=set()
bad_collections_abc_classes_in_this_import=set()
bad_contextlib_classes_in_this_import=set()
ifnode.module=="typing":
forclsinnode.names:
ifcls.nameinIMPORTED_FROM_BUILTINS_NOT_TYPING:
bad_builtins_classes_in_this_import.add(cls)
elifcls.nameinIMPORTED_FROM_COLLECTIONS_NOT_TYPING_OR_TYPING_EXTENSIONSandpath!=Path(
"stdlib/typing_extensions.pyi"
):
bad_collections_classes_in_this_import.add(cls)
elifcls.name=="AsyncContextManager":
bad_contextlib_classes_in_this_import.add(cls)
else:
forclsinnode.names:
ifcls.nameinIMPORTED_FROM_COLLECTIONS_NOT_TYPING_OR_TYPING_EXTENSIONS:
bad_collections_classes_in_this_import.add(cls)
elifcls.nameinIMPORTED_FROM_TYPING_NOT_TYPING_EXTENSIONS:
bad_collections_abc_classes_in_this_import.add(cls)
elifcls.namein {"ContextManager", "AsyncContextManager"}:
bad_contextlib_classes_in_this_import.add(cls)
bad_classes_in_this_import= (
bad_builtins_classes_in_this_import|bad_collections_classes_in_this_import|bad_collections_abc_classes_in_this_import|bad_contextlib_classes_in_this_import
)
ifnotbad_classes_in_this_import:
returnifnode.module=="typing":
classes_from_typing.update(cls.nameforclsinbad_classes_in_this_import)
else:
classes_from_typing_extensions.update(cls.nameforclsinbad_classes_in_this_import)
new_import_list= [clsforclsinnode.namesifclsnotinbad_classes_in_this_import]
### DEALING WITH EXISTING IMPORT STATEMENTS #### Scenario (1): Now we don't need *any* imports from typing/typing_extensions any more.ifnotnew_import_list:
imports_to_delete[node.lineno-1] =DeleteableImport(old=ast.unparse(node), replacement="")
# Scenario (2): we still need imports from typing/typing_extensions; the existing import statement is only one lineelifnode.lineno==node.end_lineno:
imports_to_delete[node.lineno-1] =DeleteableImport(
old=ast.unparse(node),
replacement=ast.unparse(ast.ImportFrom(module=node.module, names=new_import_list, level=0)),
)
# Scenario (3): we still need imports from typing/typing_extensions; the existing import statement is multiline.else:
forclsinnode.names:
ifclsinbad_classes_in_this_import:
imports_to_delete[cls.lineno-1] =DeleteableImport(
old=f"{cls.name},"ifcls.asnameisNoneelsef"{cls.name} as {cls.asname},", replacement=""
)
### ADDING NEW IMPORT STATEMENTS ###ifbad_collections_classes_in_this_import:
imports_to_add[node.lineno-1].append(
NewImport(
text=ast.unparse(
ast.ImportFrom(
module="collections",
names=[
ast.alias(
name=IMPORTED_FROM_COLLECTIONS_NOT_TYPING_OR_TYPING_EXTENSIONS[cls.name],
asname=cls.asname,
)
forclsinbad_collections_classes_in_this_import
],
level=0,
)
),
indentation=node.col_offset,
)
)
ifbad_collections_abc_classes_in_this_import:
imports_to_add[node.lineno-1].append(
NewImport(
text=ast.unparse(ast.ImportFrom(module="typing", names=classes_to_import, level=0)),
indentation=node.col_offset,
)
)
BadImportFinder().visit(tree)
ifnotclasses_from_typing|classes_from_typing_extensions:
returnforlineno, (old_syntax, new_syntax) inimports_to_delete.items():
lines[lineno] =lines[lineno].replace(old_syntax, new_syntax)
forlineno, import_listinimports_to_add.items():
fornew_import, indentationinimport_list:
ifisinstance(new_import, str):
lines.insert(lineno, f'{" "*indentation}{new_import}')
else:
lines=lines[:lineno] + [f'{" "*indentation}{l}'forlinnew_import] +lines[lineno:]
try:
new_tree=ast.parse("\n".join(lines))
exceptSyntaxError:
sys.stderr.write(f"Error converting new syntax in {path}")
FAILURES.append(path)
else:
lines_with_bad_syntax=defaultdict(list)
classOldSyntaxFinder(ast.NodeVisitor):
defvisit_Subscript(self, node: ast.Subscript) ->None:
ifisinstance(node.value, ast.Name) andnode.value.idin (
(classes_from_typing|classes_from_typing_extensions)
& (IMPORTED_FROM_BUILTINS_NOT_TYPING| {"Deque", "DefaultDict"})
):
lines_with_bad_syntax[node.lineno-1].append(node.value.id)
self.generic_visit(node)
OldSyntaxFinder().visit(new_tree)
fori, cls_listinlines_with_bad_syntax.items():
forclsincls_list:
lines[i] =re.sub(fr"(\W){cls}\[", fr"\1{cls.lower()}[", lines[i])
withopen(path, "w") asf:
f.write("\n".join(lines) +"\n")
defmain() ->None:
print("STARTING RUN: Will attempt to fix new syntax in typeshed directory...\n\n")
forpathinchain(Path("stdlib").rglob("*.pyi"), Path("stubs").rglob("*.pyi")):
print(f"Attempting to convert {path} to new syntax.")
fix_bad_syntax(path)
print("\n\nSTARTING ISORT...\n\n")
forfolderin {"stdlib", "stubs", "tests"}:
subprocess.run([sys.executable, "-m", "isort", folder])
print("\n\nSTARTING BLACK...\n\n")
subprocess.run([sys.executable, "-m", "black", "."])
ifFAILURES:
print("\n\nFAILED to convert the following files to new syntax:\n")
forpathinFAILURES:
print(f"- {path}")
else:
print("\n\nThere were ZERO failures in converting to new syntax. HOORAY!!\n\n")
print('\n\nRunning "check_new_syntax.py"...\n\n')
subprocess.run([sys.executable, "tests/check_new_syntax.py"])
print('\n\nRunning "stubtest_stdlib.py"...\n\n')
subprocess.run([sys.executable, "tests/stubtest_stdlib.py"])
if__name__=="__main__":
main()

[github-actions]

According to mypy_primer, this change has no effect on the checked open source code. 🤖🎉

[JelleZijlstra]

Thanks! I think we should leave protobuf alone because the stubs are autogenerated.

[AlexWaygood]

Thanks! I think we should leave protobuf alone because the stubs are autogenerated.

Okay — should I add per-file excludes for the proposed new error codes to https://github.com/python/typeshed/blob/master/.flake8?

[JelleZijlstra]

Actually, after looking at #6944 I realized these particular files are not autogenerated.

[AlexWaygood]

Thanks!