Use PEP 585 syntax wherever possible

[AlexWaygood]

This PR proposes using PEP 585 syntax where possible. There are two situations where this is currently not possible:

• Lowercase type still causes mypy errors in some situations (see, e.g., here).
• Importing from collections.abc rather than typing will often cause pytype to error (see, e.g., here).

This PR was created using the following script

importastimportreimportsubprocessimportsysfromcollectionsimportdefaultdictfromitertoolsimportchainfromoperatorimportattrgetterfrompathlibimportPathfromtypingimportNamedTupleclassDeleteableImport(NamedTuple):
old: strreplacement: strclassNewImport(NamedTuple):
text: strindentation: intFORBIDDEN_BUILTIN_TYPING_IMPORTS=frozenset({"List", "FrozenSet", "Set", "Dict", "Tuple"})
# The values in the mapping are what these are called in `collections`IMPORTED_FROM_COLLECTIONS_NOT_TYPING= {
"Counter": "Counter",
"Deque": "deque",
"DefaultDict": "defaultdict",
"OrderedDict": "OrderedDict",
"ChainMap": "ChainMap",
}
FAILURES= []
deffix_bad_syntax(path: Path) ->None:
withopen(path) asf:
stub=f.read()
lines=stub.splitlines()
tree=ast.parse(stub)
imports_to_delete= {}
imports_to_add=defaultdict(list)
classes_from_typing=set()
classBadImportFinder(ast.NodeVisitor):
defvisit_ImportFrom(self, node: ast.ImportFrom) ->None:
ifnode.module!="typing":
returnbad_builtins_classes_in_this_import=set()
bad_collections_classes_in_this_import=set()
forclsinnode.names:
ifcls.nameinFORBIDDEN_BUILTIN_TYPING_IMPORTS:
bad_builtins_classes_in_this_import.add(cls)
elifcls.nameinIMPORTED_FROM_COLLECTIONS_NOT_TYPING:
bad_collections_classes_in_this_import.add(cls)
bad_classes_in_this_import=bad_builtins_classes_in_this_import|bad_collections_classes_in_this_importifnotbad_classes_in_this_import:
returnclasses_from_typing.update(cls.nameforclsinbad_classes_in_this_import)
new_import_list= [clsforclsinnode.namesifclsnotinbad_classes_in_this_import]
### DEALING WITH EXISTING IMPORT STATEMENTS #### Scenario (1): Now we don't need *any* imports from typing any more.ifnotnew_import_list:
ifpath==Path("stdlib/csv.pyi"):
imports_to_delete[node.lineno-1] =DeleteableImport(
old=ast.unparse(node), replacement="from builtins import dict as _DictReadMapping"
)
else:
imports_to_delete[node.lineno-1] =DeleteableImport(old=ast.unparse(node), replacement="")
# Scenario (2): we still need imports from typing; the existing import statement is only one lineelifnode.lineno==node.end_lineno:
imports_to_delete[node.lineno-1] =DeleteableImport(
old=ast.unparse(node),
replacement=ast.unparse(ast.ImportFrom(module="typing", names=new_import_list, level=0)),
)
# Scenario (3): we still need imports from typing; the existing import statement is multiline.else:
forclsinnode.names:
ifclsinbad_classes_in_this_import:
imports_to_delete[cls.lineno-1] =DeleteableImport(
old=f"{cls.name},"ifcls.asnameisNoneelsef"{cls.name} as {cls.asname},", replacement=""
)
### ADDING NEW IMPORT STATEMENTS ###ifbad_collections_classes_in_this_import:
imports_to_add[node.lineno-1].append(
NewImport(
text=ast.unparse(
ast.ImportFrom(
module="collections",
names=[
ast.alias(name=IMPORTED_FROM_COLLECTIONS_NOT_TYPING[cls.name], asname=cls.asname)
forclsinsorted(bad_collections_classes_in_this_import, key=attrgetter("name"))
],
level=0,
)
),
indentation=node.col_offset,
)
)
BadImportFinder().visit(tree)
ifnotclasses_from_typing:
returnforlineno, (old_syntax, new_syntax) inimports_to_delete.items():
lines[lineno] =lines[lineno].replace(old_syntax, new_syntax)
forlineno, import_listinimports_to_add.items():
fornew_import, indentationinimport_list:
ifisinstance(new_import, str):
lines.insert(lineno, f'{" "*indentation}{new_import}')
else:
lines=lines[:lineno] + [f'{" "*indentation}{l}'forlinnew_import] +lines[lineno:]
try:
new_tree=ast.parse("\n".join(lines))
exceptSyntaxError:
sys.stderr.write(f"Error converting new syntax in {path}")
FAILURES.append(path)
else:
lines_with_bad_syntax=defaultdict(list)
classOldSyntaxFinder(ast.NodeVisitor):
defvisit_Subscript(self, node: ast.Subscript) ->None:
ifisinstance(node.value, ast.Name) andnode.value.idin (
classes_from_typing& (FORBIDDEN_BUILTIN_TYPING_IMPORTS| {"Deque", "DefaultDict"})
):
lines_with_bad_syntax[node.lineno-1].append(node.value.id)
self.generic_visit(node)
OldSyntaxFinder().visit(new_tree)
fori, cls_listinlines_with_bad_syntax.items():
forclsincls_list:
lines[i] =re.sub(fr"(\W){cls}\[", fr"\1{cls.lower()}[", lines[i])
new_stub="\n".join(lines) +"\n"ifpath==Path("stdlib/plistlib.pyi"):
new_stub=new_stub.replace("_Dict", "dict")
withopen(path, "w") asf:
f.write(new_stub)
defmain() ->None:
print("STARTING RUN: Will attempt to fix new syntax in typeshed directory...\n\n")
forpathinchain(Path("stdlib").rglob("*.pyi"), Path("stubs").rglob("*.pyi")):
if"@python2"inpath.parts:
print(f"Skipping {path}: Python-2 stub")
elifPath("stubs/protobuf/google/protobuf") inpath.parents:
print(f"Skipping {path}: protobuf stub")
else:
print(f"Attempting to convert {path} to new syntax.")
fix_bad_syntax(path)
print("\n\nSTARTING ISORT...\n\n")
forfolderin {"stdlib", "stubs", "tests"}:
subprocess.run([sys.executable, "-m", "isort", folder])
print("\n\nSTARTING BLACK...\n\n")
subprocess.run([sys.executable, "-m", "black", "."])
ifFAILURES:
print("\n\nFAILED to convert the following files to new syntax:\n")
forpathinFAILURES:
print(f"- {path}")
else:
print("\n\nThere were ZERO failures in converting to new syntax. HOORAY!!\n\n")
print('\n\nRunning "check_new_syntax.py"...\n\n')
subprocess.run([sys.executable, "tests/check_new_syntax.py"])
print('\n\nRunning "stubtest_stdlib.py"...\n\n')
subprocess.run([sys.executable, "tests/stubtest_stdlib.py"])
if__name__=="__main__":
main()

[github-actions]

According to mypy_primer, this change has no effect on the checked open source code. 🤖🎉

[srittau]

Spot checking showed no problems. Apart from that, I trust our tests.

[AlexWaygood]

Thanks @srittau! 😀