Skip to content

Make collections.abcs more consistent with runtime implementation#10816

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JelleZijlstra merged 15 commits into from
Dec 28, 2024
Merged

Make collections.abcs more consistent with runtime implementation #10816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
78a77de
Make more consistent with runtime implementation
Gobot1234 Oct 1, 2023
0fd644f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 1, 2023
049a270
mark as runtime checkable
Gobot1234 Oct 1, 2023
ab4613b
This might help a smidge
Gobot1234 Oct 1, 2023
0d7868f
Merge branch 'main' into types-over-collections.abc
JelleZijlstra Oct 2, 2024
c7fead5
Update stdlib/typing.pyi
JelleZijlstra Oct 2, 2024
92782a2
Update common.txt
Gobot1234 Oct 2, 2024
cab10a8
Update py38.txt
Gobot1234 Oct 2, 2024
255b6f0
Update py39.txt
Gobot1234 Oct 2, 2024
bf8700f
Merge branch 'main' into types-over-collections.abc
JelleZijlstra Oct 17, 2024
a2ce3b2
Merge branch 'main' into types-over-collections.abc
JelleZijlstra Dec 28, 2024
ab12c5e
Merge remote-tracking branch 'upstream/main' into types-over-collecti…
JelleZijlstra Dec 28, 2024
fae5bed
fixup
JelleZijlstra Dec 28, 2024
e1673b4
fix more
JelleZijlstra Dec 28, 2024
2cd7e8f
put more back
JelleZijlstra Dec 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions stdlib/@tests/stubtest_allowlists/common.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,11 +4,6 @@

# Please keep sorted alphabetically

_collections_abc.AsyncGenerator.ag_await
_collections_abc.AsyncGenerator.ag_code
_collections_abc.AsyncGenerator.ag_frame
_collections_abc.AsyncGenerator.ag_running
asyncio.__all__
builtins.dict.get
collections\.ChainMap\.fromkeys # https://github.com/python/mypy/issues/17023
http.client.HTTPConnection.response_class # the actual type at runtime is abc.ABCMeta
Expand DownExpand Up@@ -221,16 +216,6 @@ _collections_abc.AsyncIterator.__anext__
_collections_abc.ByteString

_collections_abc.Callable # Typing-related weirdness

# Coroutine and Generator properties are added programmatically
_collections_abc.Coroutine.cr_await
_collections_abc.Coroutine.cr_code
_collections_abc.Coroutine.cr_frame
_collections_abc.Coroutine.cr_running
_collections_abc.Generator.gi_code
_collections_abc.Generator.gi_frame
_collections_abc.Generator.gi_running
_collections_abc.Generator.gi_yieldfrom
_collections_abc.Mapping.get # Adding None to the Union messed up mypy
_collections_abc.Sequence.index # Supporting None in end is not mandatory

Expand Down
12 changes: 0 additions & 12 deletions stdlib/@tests/stubtest_allowlists/py38.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,22 +59,10 @@ tkinter.tix.wantobjects
builtins.input # Incorrect default value in text signature, fixed in 3.10
collections.AsyncGenerator.__anext__ # async at runtime, deliberately not in the stub, see #7491
collections.AsyncGenerator.aclose # async at runtime, deliberately not in the stub, see #7491
collections.AsyncGenerator.ag_await
collections.AsyncGenerator.ag_code
collections.AsyncGenerator.ag_frame
collections.AsyncGenerator.ag_running
collections.AsyncGenerator.asend # async at runtime, deliberately not in the stub, see #7491. Pos-only differences also.
collections.AsyncIterator.__anext__ # async at runtime, deliberately not in the stub, see #7491
collections.ByteString # see comments in py3_common.txt
collections.Callable
collections.Coroutine.cr_await
collections.Coroutine.cr_code
collections.Coroutine.cr_frame
collections.Coroutine.cr_running
collections.Generator.gi_code
collections.Generator.gi_frame
collections.Generator.gi_running
collections.Generator.gi_yieldfrom
collections.Mapping.get # Adding None to the Union messed up mypy
collections.Sequence.index # Supporting None in end is not mandatory
xxsubtype # module missing from the stubs
Expand Down
13 changes: 0 additions & 13 deletions stdlib/@tests/stubtest_allowlists/py39.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,22 +50,10 @@ typing._SpecialForm.__mro_entries__ # Exists at runtime, but missing from stubs
builtins.input # Incorrect default value in text signature, fixed in 3.10
collections.AsyncGenerator.__anext__ # async at runtime, deliberately not in the stub, see #7491
collections.AsyncGenerator.aclose # async at runtime, deliberately not in the stub, see #7491
collections.AsyncGenerator.ag_await
collections.AsyncGenerator.ag_code
collections.AsyncGenerator.ag_frame
collections.AsyncGenerator.ag_running
collections.AsyncGenerator.asend # async at runtime, deliberately not in the stub, see #7491. Pos-only differences also.
collections.AsyncIterator.__anext__ # async at runtime, deliberately not in the stub, see #7491
collections.ByteString # see comments in py3_common.txt
collections.Callable
collections.Coroutine.cr_await
collections.Coroutine.cr_code
collections.Coroutine.cr_frame
collections.Coroutine.cr_running
collections.Generator.gi_code
collections.Generator.gi_frame
collections.Generator.gi_running
collections.Generator.gi_yieldfrom
collections.Mapping.get # Adding None to the Union messed up mypy
collections.Sequence.index # Supporting None in end is not mandatory
xxsubtype # module missing from the stubs
Expand DownExpand Up@@ -141,7 +129,6 @@ tkinter.tix.Shell
tkinter.tix.TclVersion
tkinter.tix.TkVersion


# ============================================================
# Allowlist entries that cannot or should not be fixed; >= 3.9
# ============================================================
Expand Down
20 changes: 20 additions & 0 deletions stdlib/types.pyi
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -374,6 +374,12 @@ _ReturnT_co = TypeVar("_ReturnT_co", covariant=True)

@final
class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]):

@srittausrittauOct 2, 2023•
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd be interested what happens if we just change types.pyi, but remove the typing base classes to better match runtime.

@Gobot1234Gobot1234Oct 2, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not entirely sure what you're proposing, wouldn't that just crash mypy because typing.Generator wouldn't exist?

@srittausrittauOct 2, 2023•
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I meant just using class GeneratorType: here (and adding the methods), not deleting typing.Generator.

@Gobot1234Gobot1234Oct 2, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typing.Generator hasn't been deleted it's just a protocol now

@property
def gi_code(self) -> CodeType: ...
@property
def gi_frame(self) -> FrameType: ...
@property
def gi_running(self) -> bool: ...
@property
def gi_yieldfrom(self) -> GeneratorType[_YieldT_co, _SendT_contra, Any] | None: ...
if sys.version_info >= (3, 11):
Expand All@@ -397,6 +403,12 @@ class GeneratorType(Generator[_YieldT_co, _SendT_contra, _ReturnT_co]):
class AsyncGeneratorType(AsyncGenerator[_YieldT_co, _SendT_contra]):
@property
def ag_await(self) -> Awaitable[Any] | None: ...
@property
def ag_code(self) -> CodeType: ...
@property
def ag_frame(self) -> FrameType: ...
@property
def ag_running(self) -> bool: ...
__name__: str
__qualname__: str
if sys.version_info >= (3, 12):
Expand All@@ -421,6 +433,14 @@ class CoroutineType(Coroutine[_YieldT_co, _SendT_contra, _ReturnT_co]):
__name__: str
__qualname__: str
@property
def cr_await(self) -> Any | None: ...
@property
def cr_code(self) -> CodeType: ...
@property
def cr_frame(self) -> FrameType: ...
@property
def cr_running(self) -> bool: ...
@property
def cr_origin(self) -> tuple[tuple[str, int, str], ...] | None: ...
if sys.version_info >= (3, 11):
@property
Expand Down
32 changes: 5 additions & 27 deletions stdlib/typing.pyi
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,6 @@ from re import Match as Match, Pattern as Pattern
from types import (
BuiltinFunctionType,
CodeType,
FrameType,
FunctionType,
MethodDescriptorType,
MethodType,
Expand DownExpand Up@@ -473,7 +472,8 @@ _YieldT_co = TypeVar("_YieldT_co", covariant=True)
_SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None)
_ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None)

class Generator(Iterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra, _ReturnT_co]):
@runtime_checkable
class Generator(Iterator[_YieldT_co], Protocol[_YieldT_co, _SendT_contra, _ReturnT_co]):
def __next__(self) -> _YieldT_co: ...
@abstractmethod
def send(self, value: _SendT_contra, /) -> _YieldT_co: ...
Expand All@@ -491,14 +491,6 @@ class Generator(Iterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra, _Return
def close(self) -> None: ...

def __iter__(self) -> Generator[_YieldT_co, _SendT_contra, _ReturnT_co]: ...

@Gobot1234Gobot1234Oct 2, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm unsure as to whether or not this should be -> Self the runtime has it as that in Iterator

@Gobot1234Gobot1234Oct 3, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Upon further consideration, not implementing it as returning Self is probably a bug so I'll change it in (Async)Iterator even

@AlexWaygoodAlexWaygoodOct 3, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

typing.Generator.__iter__ shouldn't be annotated as returning Self for the same reason that typing.Iterator.__iter__ shouldn't be annotated as returning Self. 99% of all iterators have __iter__ methods that return Self, and 99% of generators are the same. But if we hinted them as such in the stub, we'd be changing the very definition of Iterator/Generator such that the remaining 1% of iterators and generators would no longer be considered iterators or generators. And that wouldn't be accurate: it's not part of the contract that iterators have to return self from their __iter__ methods at runtime in order for them to be considered iterators, they just have to return an iterable object.

With types.GeneratorType, it's a different issue, because that class doesn't define an interface in the same way. We can just reflect what happens at runtime there.

@JelleZijlstraJelleZijlstraOct 2, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suspect removing this method is causing the mypy-primer fallout. Let's add it back.

@property
def gi_code(self) -> CodeType: ...
@property
def gi_frame(self) -> FrameType: ...
@property
def gi_running(self) -> bool: ...
@property
def gi_yieldfrom(self) -> Generator[Any, Any, Any] | None: ...

# NOTE: Prior to Python 3.13 these aliases are lacking the second _ExitT_co parameter
if sys.version_info >= (3, 13):
Expand All@@ -524,14 +516,7 @@ _ReturnT_co_nd = TypeVar("_ReturnT_co_nd", covariant=True)
class Coroutine(Awaitable[_ReturnT_co_nd], Generic[_YieldT_co, _SendT_contra_nd, _ReturnT_co_nd]):
__name__: str
__qualname__: str
@property
def cr_await(self) -> Any | None: ...
@property
def cr_code(self) -> CodeType: ...
@property
def cr_frame(self) -> FrameType | None: ...
@property
def cr_running(self) -> bool: ...

@abstractmethod
def send(self, value: _SendT_contra_nd, /) -> _YieldT_co: ...
@overload
Expand DownExpand Up@@ -566,7 +551,8 @@ class AsyncIterator(AsyncIterable[_T_co], Protocol[_T_co]):
def __anext__(self) -> Awaitable[_T_co]: ...
def __aiter__(self) -> AsyncIterator[_T_co]: ...

class AsyncGenerator(AsyncIterator[_YieldT_co], Generic[_YieldT_co, _SendT_contra]):
@runtime_checkable
class AsyncGenerator(AsyncIterator[_YieldT_co], Protocol[_YieldT_co, _SendT_contra]):
def __anext__(self) -> Coroutine[Any, Any, _YieldT_co]: ...
@abstractmethod
def asend(self, value: _SendT_contra, /) -> Coroutine[Any, Any, _YieldT_co]: ...
Expand All@@ -581,14 +567,6 @@ class AsyncGenerator(AsyncIterator[_YieldT_co], Generic[_YieldT_co, _SendT_contr
self, typ: BaseException, val: None = None, tb: TracebackType | None = None, /
) -> Coroutine[Any, Any, _YieldT_co]: ...
def aclose(self) -> Coroutine[Any, Any, None]: ...
@property
def ag_await(self) -> Any: ...
@property
def ag_code(self) -> CodeType: ...
@property
def ag_frame(self) -> FrameType: ...
@property
def ag_running(self) -> bool: ...

@runtime_checkable
class Container(Protocol[_T_co]):
Expand Down